{"nbformat":4,"nbformat_minor":0,"metadata":{"colab":{"provenance":[],"authorship_tag":"ABX9TyP8gU6idIpcP+baxXhOolrV"},"kernelspec":{"name":"python3","display_name":"Python 3"},"language_info":{"name":"python"},"widgets":{"application/vnd.jupyter.widget-state+json":{"ee48e9b394774ff8b244f678a3352cf4":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_b7002ee8d40342d19210dcc548c27aa4","IPY_MODEL_856aa125e113453bbd026f20ee843908","IPY_MODEL_522f69cbc1bf4e5d88b4fd97a4d219a4"],"layout":"IPY_MODEL_888e6958ac8b4d2d858a9688e16dd830"}},"b7002ee8d40342d19210dcc548c27aa4":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_896c46701ef942ad87d0116602744dbc","placeholder":"​","style":"IPY_MODEL_40435e30966f44efa7fea1c84b054e09","value":""}},"856aa125e113453bbd026f20ee843908":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_f083317683424036951bd56a6e307167","max":1,"min":0,"orientation":"horizontal","style":"IPY_MODEL_e5b809b105854b349b0252be952c4e95","value":1}},"522f69cbc1bf4e5d88b4fd97a4d219a4":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_4a1cfb40a74c45969ee1475e2376ee1c","placeholder":"​","style":"IPY_MODEL_aad2458fc6a24e6a8f5038647be8bde5","value":" 20/? [01:40&lt;00:00,  1.56it/s]"}},"888e6958ac8b4d2d858a9688e16dd830":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"896c46701ef942ad87d0116602744dbc":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"40435e30966f44efa7fea1c84b054e09":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"f083317683424036951bd56a6e307167":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":"20px"}},"e5b809b105854b349b0252be952c4e95":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"4a1cfb40a74c45969ee1475e2376ee1c":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"aad2458fc6a24e6a8f5038647be8bde5":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"546b22a963f54805960c31a10c1ad369":{"model_module":"@jupyter-widgets/controls","model_name":"HBoxModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HBoxModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HBoxView","box_style":"","children":["IPY_MODEL_3581bd030e4349fd8eb740ba0052d353","IPY_MODEL_fca139cfd6e5451ea214624fe965b7ce","IPY_MODEL_0ce92fcac1dc4b7fac64c21d28fcaced"],"layout":"IPY_MODEL_f88b7dc9c1ad49f2aaf19fb9920260cc"}},"3581bd030e4349fd8eb740ba0052d353":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_19809f7164fd4dedaef209d293c8f6a8","placeholder":"​","style":"IPY_MODEL_1b5a77df387c47869c71fb283b9502bc","value":""}},"fca139cfd6e5451ea214624fe965b7ce":{"model_module":"@jupyter-widgets/controls","model_name":"FloatProgressModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"FloatProgressModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"ProgressView","bar_style":"success","description":"","description_tooltip":null,"layout":"IPY_MODEL_42229b1d4a904fa895873eeb09b118bd","max":1,"min":0,"orientation":"horizontal","style":"IPY_MODEL_2586da556b7045f0880eb73b8302fe6d","value":1}},"0ce92fcac1dc4b7fac64c21d28fcaced":{"model_module":"@jupyter-widgets/controls","model_name":"HTMLModel","model_module_version":"1.5.0","state":{"_dom_classes":[],"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"HTMLModel","_view_count":null,"_view_module":"@jupyter-widgets/controls","_view_module_version":"1.5.0","_view_name":"HTMLView","description":"","description_tooltip":null,"layout":"IPY_MODEL_8948252684574d2596efeb4d519ec253","placeholder":"​","style":"IPY_MODEL_5ec9591d10eb446b8ed2fe87b5220b63","value":" 20/? [01:13&lt;00:00,  2.13it/s]"}},"f88b7dc9c1ad49f2aaf19fb9920260cc":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"19809f7164fd4dedaef209d293c8f6a8":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"1b5a77df387c47869c71fb283b9502bc":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}},"42229b1d4a904fa895873eeb09b118bd":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":"20px"}},"2586da556b7045f0880eb73b8302fe6d":{"model_module":"@jupyter-widgets/controls","model_name":"ProgressStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"ProgressStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","bar_color":null,"description_width":""}},"8948252684574d2596efeb4d519ec253":{"model_module":"@jupyter-widgets/base","model_name":"LayoutModel","model_module_version":"1.2.0","state":{"_model_module":"@jupyter-widgets/base","_model_module_version":"1.2.0","_model_name":"LayoutModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"LayoutView","align_content":null,"align_items":null,"align_self":null,"border":null,"bottom":null,"display":null,"flex":null,"flex_flow":null,"grid_area":null,"grid_auto_columns":null,"grid_auto_flow":null,"grid_auto_rows":null,"grid_column":null,"grid_gap":null,"grid_row":null,"grid_template_areas":null,"grid_template_columns":null,"grid_template_rows":null,"height":null,"justify_content":null,"justify_items":null,"left":null,"margin":null,"max_height":null,"max_width":null,"min_height":null,"min_width":null,"object_fit":null,"object_position":null,"order":null,"overflow":null,"overflow_x":null,"overflow_y":null,"padding":null,"right":null,"top":null,"visibility":null,"width":null}},"5ec9591d10eb446b8ed2fe87b5220b63":{"model_module":"@jupyter-widgets/controls","model_name":"DescriptionStyleModel","model_module_version":"1.5.0","state":{"_model_module":"@jupyter-widgets/controls","_model_module_version":"1.5.0","_model_name":"DescriptionStyleModel","_view_count":null,"_view_module":"@jupyter-widgets/base","_view_module_version":"1.2.0","_view_name":"StyleView","description_width":""}}}}},"cells":[{"cell_type":"code","execution_count":null,"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"collapsed":true,"id":"ztExTsdJ6eXC","executionInfo":{"status":"ok","timestamp":1724804752732,"user_tz":-480,"elapsed":26574,"user":{"displayName":"李辉","userId":"12972001611808140221"}},"outputId":"fdd149f6-50fd-43cd-96e7-aa09500a2735"},"outputs":[{"output_type":"stream","name":"stdout","text":["Collecting langchain_community\n","  Using cached langchain_community-0.2.12-py3-none-any.whl.metadata (2.7 kB)\n","Collecting langchain-openai\n","  Using cached langchain_openai-0.1.22-py3-none-any.whl.metadata (2.6 kB)\n","Collecting langchain\n","  Using cached langchain-0.2.14-py3-none-any.whl.metadata (7.1 kB)\n","Collecting langgraph\n","  Downloading langgraph-0.2.14-py3-none-any.whl.metadata (13 kB)\n","Collecting bs4\n","  Downloading bs4-0.0.2-py2.py3-none-any.whl.metadata (411 bytes)\n","Requirement already satisfied: PyYAML>=5.3 in /usr/local/lib/python3.10/dist-packages (from langchain_community) (6.0.2)\n","Requirement already satisfied: SQLAlchemy<3,>=1.4 in /usr/local/lib/python3.10/dist-packages (from langchain_community) (2.0.32)\n","Requirement already satisfied: aiohttp<4.0.0,>=3.8.3 in /usr/local/lib/python3.10/dist-packages (from langchain_community) (3.10.5)\n","Collecting dataclasses-json<0.7,>=0.5.7 (from langchain_community)\n","  Downloading dataclasses_json-0.6.7-py3-none-any.whl.metadata (25 kB)\n","Collecting langchain-core<0.3.0,>=0.2.30 (from langchain_community)\n","  Downloading langchain_core-0.2.35-py3-none-any.whl.metadata (6.2 kB)\n","Collecting langsmith<0.2.0,>=0.1.0 (from langchain_community)\n","  Downloading langsmith-0.1.106-py3-none-any.whl.metadata (13 kB)\n","Requirement already satisfied: numpy<2,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain_community) (1.26.4)\n","Requirement already satisfied: requests<3,>=2 in /usr/local/lib/python3.10/dist-packages (from langchain_community) (2.32.3)\n","Collecting tenacity!=8.4.0,<9.0.0,>=8.1.0 (from langchain_community)\n","  Downloading tenacity-8.5.0-py3-none-any.whl.metadata (1.2 kB)\n","Collecting openai<2.0.0,>=1.40.0 (from langchain-openai)\n","  Downloading openai-1.42.0-py3-none-any.whl.metadata (22 kB)\n","Collecting tiktoken<1,>=0.7 (from langchain-openai)\n","  Downloading tiktoken-0.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (6.6 kB)\n","Requirement already satisfied: async-timeout<5.0.0,>=4.0.0 in /usr/local/lib/python3.10/dist-packages (from langchain) (4.0.3)\n","Collecting langchain-text-splitters<0.3.0,>=0.2.0 (from langchain)\n","  Downloading langchain_text_splitters-0.2.2-py3-none-any.whl.metadata (2.1 kB)\n","Requirement already satisfied: pydantic<3,>=1 in /usr/local/lib/python3.10/dist-packages (from langchain) (2.8.2)\n","Collecting langgraph-checkpoint<2.0.0,>=1.0.2 (from langgraph)\n","  Downloading langgraph_checkpoint-1.0.6-py3-none-any.whl.metadata (4.5 kB)\n","Requirement already satisfied: beautifulsoup4 in /usr/local/lib/python3.10/dist-packages (from bs4) (4.12.3)\n","Requirement already satisfied: aiohappyeyeballs>=2.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain_community) (2.4.0)\n","Requirement already satisfied: aiosignal>=1.1.2 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain_community) (1.3.1)\n","Requirement already satisfied: attrs>=17.3.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain_community) (24.2.0)\n","Requirement already satisfied: frozenlist>=1.1.1 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain_community) (1.4.1)\n","Requirement already satisfied: multidict<7.0,>=4.5 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain_community) (6.0.5)\n","Requirement already satisfied: yarl<2.0,>=1.0 in /usr/local/lib/python3.10/dist-packages (from aiohttp<4.0.0,>=3.8.3->langchain_community) (1.9.4)\n","Collecting marshmallow<4.0.0,>=3.18.0 (from dataclasses-json<0.7,>=0.5.7->langchain_community)\n","  Downloading marshmallow-3.22.0-py3-none-any.whl.metadata (7.2 kB)\n","Collecting typing-inspect<1,>=0.4.0 (from dataclasses-json<0.7,>=0.5.7->langchain_community)\n","  Downloading typing_inspect-0.9.0-py3-none-any.whl.metadata (1.5 kB)\n","Collecting jsonpatch<2.0,>=1.33 (from langchain-core<0.3.0,>=0.2.30->langchain_community)\n","  Downloading jsonpatch-1.33-py2.py3-none-any.whl.metadata (3.0 kB)\n","Requirement already satisfied: packaging<25,>=23.2 in /usr/local/lib/python3.10/dist-packages (from langchain-core<0.3.0,>=0.2.30->langchain_community) (24.1)\n","Requirement already satisfied: typing-extensions>=4.7 in /usr/local/lib/python3.10/dist-packages (from langchain-core<0.3.0,>=0.2.30->langchain_community) (4.12.2)\n","Collecting httpx<1,>=0.23.0 (from langsmith<0.2.0,>=0.1.0->langchain_community)\n","  Downloading httpx-0.27.2-py3-none-any.whl.metadata (7.1 kB)\n","Collecting orjson<4.0.0,>=3.9.14 (from langsmith<0.2.0,>=0.1.0->langchain_community)\n","  Downloading orjson-3.10.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (50 kB)\n","\u001b[2K     \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m50.4/50.4 kB\u001b[0m \u001b[31m500.2 kB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hRequirement already satisfied: anyio<5,>=3.5.0 in /usr/local/lib/python3.10/dist-packages (from openai<2.0.0,>=1.40.0->langchain-openai) (3.7.1)\n","Requirement already satisfied: distro<2,>=1.7.0 in /usr/lib/python3/dist-packages (from openai<2.0.0,>=1.40.0->langchain-openai) (1.7.0)\n","Collecting jiter<1,>=0.4.0 (from openai<2.0.0,>=1.40.0->langchain-openai)\n","  Downloading jiter-0.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl.metadata (3.6 kB)\n","Requirement already satisfied: sniffio in /usr/local/lib/python3.10/dist-packages (from openai<2.0.0,>=1.40.0->langchain-openai) (1.3.1)\n","Requirement already satisfied: tqdm>4 in /usr/local/lib/python3.10/dist-packages (from openai<2.0.0,>=1.40.0->langchain-openai) (4.66.5)\n","Requirement already satisfied: annotated-types>=0.4.0 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1->langchain) (0.7.0)\n","Requirement already satisfied: pydantic-core==2.20.1 in /usr/local/lib/python3.10/dist-packages (from pydantic<3,>=1->langchain) (2.20.1)\n","Requirement already satisfied: charset-normalizer<4,>=2 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain_community) (3.3.2)\n","Requirement already satisfied: idna<4,>=2.5 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain_community) (3.7)\n","Requirement already satisfied: urllib3<3,>=1.21.1 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain_community) (2.0.7)\n","Requirement already satisfied: certifi>=2017.4.17 in /usr/local/lib/python3.10/dist-packages (from requests<3,>=2->langchain_community) (2024.7.4)\n","Requirement already satisfied: greenlet!=0.4.17 in /usr/local/lib/python3.10/dist-packages (from SQLAlchemy<3,>=1.4->langchain_community) (3.0.3)\n","Requirement already satisfied: regex>=2022.1.18 in /usr/local/lib/python3.10/dist-packages (from tiktoken<1,>=0.7->langchain-openai) (2024.5.15)\n","Requirement already satisfied: soupsieve>1.2 in /usr/local/lib/python3.10/dist-packages (from beautifulsoup4->bs4) (2.6)\n","Requirement already satisfied: exceptiongroup in /usr/local/lib/python3.10/dist-packages (from anyio<5,>=3.5.0->openai<2.0.0,>=1.40.0->langchain-openai) (1.2.2)\n","Collecting httpcore==1.* (from httpx<1,>=0.23.0->langsmith<0.2.0,>=0.1.0->langchain_community)\n","  Downloading httpcore-1.0.5-py3-none-any.whl.metadata (20 kB)\n","Collecting h11<0.15,>=0.13 (from httpcore==1.*->httpx<1,>=0.23.0->langsmith<0.2.0,>=0.1.0->langchain_community)\n","  Downloading h11-0.14.0-py3-none-any.whl.metadata (8.2 kB)\n","Collecting jsonpointer>=1.9 (from jsonpatch<2.0,>=1.33->langchain-core<0.3.0,>=0.2.30->langchain_community)\n","  Downloading jsonpointer-3.0.0-py2.py3-none-any.whl.metadata (2.3 kB)\n","Collecting mypy-extensions>=0.3.0 (from typing-inspect<1,>=0.4.0->dataclasses-json<0.7,>=0.5.7->langchain_community)\n","  Downloading mypy_extensions-1.0.0-py3-none-any.whl.metadata (1.1 kB)\n","Downloading langchain_community-0.2.12-py3-none-any.whl (2.3 MB)\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m2.3/2.3 MB\u001b[0m \u001b[31m12.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hDownloading langchain_openai-0.1.22-py3-none-any.whl (51 kB)\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m52.0/52.0 kB\u001b[0m \u001b[31m2.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hDownloading langchain-0.2.14-py3-none-any.whl (997 kB)\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m997.8/997.8 kB\u001b[0m \u001b[31m31.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hDownloading langgraph-0.2.14-py3-none-any.whl (87 kB)\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m87.7/87.7 kB\u001b[0m \u001b[31m4.3 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hDownloading bs4-0.0.2-py2.py3-none-any.whl (1.2 kB)\n","Downloading dataclasses_json-0.6.7-py3-none-any.whl (28 kB)\n","Downloading langchain_core-0.2.35-py3-none-any.whl (394 kB)\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m394.9/394.9 kB\u001b[0m \u001b[31m22.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hDownloading langchain_text_splitters-0.2.2-py3-none-any.whl (25 kB)\n","Downloading langgraph_checkpoint-1.0.6-py3-none-any.whl (15 kB)\n","Downloading langsmith-0.1.106-py3-none-any.whl (150 kB)\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m150.6/150.6 kB\u001b[0m \u001b[31m9.4 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hDownloading openai-1.42.0-py3-none-any.whl (362 kB)\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m362.9/362.9 kB\u001b[0m \u001b[31m19.1 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hDownloading tenacity-8.5.0-py3-none-any.whl (28 kB)\n","Downloading tiktoken-0.7.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (1.1 MB)\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m1.1/1.1 MB\u001b[0m \u001b[31m23.7 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hDownloading httpx-0.27.2-py3-none-any.whl (76 kB)\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m76.4/76.4 kB\u001b[0m \u001b[31m3.9 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hDownloading httpcore-1.0.5-py3-none-any.whl (77 kB)\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m77.9/77.9 kB\u001b[0m \u001b[31m4.2 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hDownloading jiter-0.5.0-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (318 kB)\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m318.9/318.9 kB\u001b[0m \u001b[31m12.8 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hDownloading jsonpatch-1.33-py2.py3-none-any.whl (12 kB)\n","Downloading marshmallow-3.22.0-py3-none-any.whl (49 kB)\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m49.3/49.3 kB\u001b[0m \u001b[31m1.5 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hDownloading orjson-3.10.7-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl (141 kB)\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m141.9/141.9 kB\u001b[0m \u001b[31m7.0 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hDownloading typing_inspect-0.9.0-py3-none-any.whl (8.8 kB)\n","Downloading jsonpointer-3.0.0-py2.py3-none-any.whl (7.6 kB)\n","Downloading mypy_extensions-1.0.0-py3-none-any.whl (4.7 kB)\n","Downloading h11-0.14.0-py3-none-any.whl (58 kB)\n","\u001b[2K   \u001b[90m━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━\u001b[0m \u001b[32m58.3/58.3 kB\u001b[0m \u001b[31m2.6 MB/s\u001b[0m eta \u001b[36m0:00:00\u001b[0m\n","\u001b[?25hInstalling collected packages: tenacity, orjson, mypy-extensions, marshmallow, jsonpointer, jiter, h11, typing-inspect, tiktoken, jsonpatch, httpcore, bs4, httpx, dataclasses-json, openai, langsmith, langchain-core, langgraph-checkpoint, langchain-text-splitters, langchain-openai, langgraph, langchain, langchain_community\n","  Attempting uninstall: tenacity\n","    Found existing installation: tenacity 9.0.0\n","    Uninstalling tenacity-9.0.0:\n","      Successfully uninstalled tenacity-9.0.0\n","Successfully installed bs4-0.0.2 dataclasses-json-0.6.7 h11-0.14.0 httpcore-1.0.5 httpx-0.27.2 jiter-0.5.0 jsonpatch-1.33 jsonpointer-3.0.0 langchain-0.2.14 langchain-core-0.2.35 langchain-openai-0.1.22 langchain-text-splitters-0.2.2 langchain_community-0.2.12 langgraph-0.2.14 langgraph-checkpoint-1.0.6 langsmith-0.1.106 marshmallow-3.22.0 mypy-extensions-1.0.0 openai-1.42.0 orjson-3.10.7 tenacity-8.5.0 tiktoken-0.7.0 typing-inspect-0.9.0\n"]}],"source":["! pip install -U langchain_community langchain-openai langchain langgraph bs4"]},{"cell_type":"code","source":["import os\n","from google.colab import userdata\n","os.environ[\"OPENAI_API_KEY\"] = userdata.get('OPENAI_API_KEY')\n","os.environ[\"OPENAI_API_BASE\"] = userdata.get('OPENAI_API_BASE')\n","os.environ[\"LANGCHAIN_TRACING_V2\"] = \"true\"\n","os.environ[\"LANGCHAIN_API_KEY\"] = userdata.get('LANGCHAIN_API_KEY')\n","os.environ[\"LANGCHAIN_PROJECT\"] = \"LangGraph Tutorial\""],"metadata":{"id":"FWx3ef8x8SrA"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["from bs4 import BeautifulSoup as Soup\n","from langchain_community.document_loaders.recursive_url_loader import RecursiveUrlLoader\n","\n","# LCEL docs\n","url = \"https://python.langchain.com/v0.2/docs/concepts/#langchain-expression-language-lcel\"\n","loader = RecursiveUrlLoader(\n","    url=url, max_depth=20, extractor=lambda x: Soup(x, \"html.parser\").text\n",")\n","docs = loader.load()\n","\n","# Sort the list based on the URLs and get the text\n","d_sorted = sorted(docs, key=lambda x: x.metadata[\"source\"])\n","d_reversed = list(reversed(d_sorted))\n","concatenated_content = \"\\n\\n\\n --- \\n\\n\\n\".join(\n","    [doc.page_content for doc in d_reversed]\n",")\n","concatenated_content"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":122},"id":"cB4OdG1E7wa6","executionInfo":{"status":"ok","timestamp":1724805227663,"user_tz":-480,"elapsed":3894,"user":{"displayName":"李辉","userId":"12972001611808140221"}},"outputId":"109995f1-8cb3-4cb1-92c9-e6ac68071cb5"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["'\\n\\n\\n\\n\\nConceptual guide | \\uf8ffü¶úÔ∏è\\uf8ffüîó LangChain\\n\\n\\n\\n\\n\\n\\n\\nSkip to main contentLangChain 0.2 is out! Leave feedback on the v0.2 docs here. You can view the v0.1 docs here.IntegrationsAPI referenceLatestLegacyMorePeopleContributingCookbooks3rd party tutorialsYouTubearXivv0.2v0.2v0.1\\uf8ffü¶úÔ∏è\\uf8ffüîóLangSmithLangSmith DocsLangChain HubJS/TS Docs\\uf8ffüí¨SearchIntroductionTutorialsBuild a Question Answering application over a Graph DatabaseTutorialsBuild a Simple LLM Application with LCELBuild a Query Analysis SystemBuild a ChatbotConversational RAGBuild an Extraction ChainBuild an AgentTaggingdata_generationBuild a Local RAG ApplicationBuild a PDF ingestion and Question/Answering systemBuild a Retrieval Augmented Generation (RAG) AppVector stores and retrieversBuild a Question/Answering system over SQL dataSummarize TextHow-to guidesHow-to guidesHow to use tools in a chainHow to use a vectorstore as a retrieverHow to add memory to chatbotsHow to use example selectorsHow to map values to a graph databaseHow to add a semantic layer over graph databaseHow to invoke runnables in parallelHow to stream chat model responsesHow to add default invocation args to a RunnableHow to add retrieval to chatbotsHow to use few shot examples in chat modelsHow to do tool/function callingHow to best prompt for Graph-RAGHow to install LangChain packagesHow to add examples to the prompt for query analysisHow to use few shot examplesHow to run custom functionsHow to use output parsers to parse an LLM response into structured formatHow to handle cases where no queries are generatedHow to route between sub-chainsHow to return structured data from a modelHow to summarize text through parallelizationHow to summarize text through iterative refinementHow to summarize text in a single LLM callHow to use toolkitsHow to add ad-hoc tool calling capability to LLMs and Chat ModelsBuild an Agent with AgentExecutor (Legacy)How to construct knowledge graphsHow to partially format prompt templatesHow to handle multiple queries when doing query analysisHow to use built-in tools and toolkitsHow to pass through arguments from one step to the nextHow to compose prompts togetherHow to handle multiple retrievers when doing query analysisHow to add values to a chain\\'s stateHow to construct filters for query analysisHow to configure runtime chain internalsHow deal with high cardinality categoricals when doing query analysisCustom Document LoaderHow to split by HTML headerHow to split by HTML sectionsHow to use the MultiQueryRetrieverHow to add scores to retriever resultsCachingHow to use callbacks in async environmentsHow to attach callbacks to a runnableHow to propagate callbacks  constructorHow to dispatch custom callback eventsHow to pass callbacks in at runtimeHow to split by characterHow to cache chat model responsesHow to handle rate limitsHow to init any model in one lineHow to track token usage in ChatModelsHow to add tools to chatbotsHow to split codeHow to do retrieval with contextual compressionHow to convert Runnables as ToolsHow to create custom callback handlersHow to create a custom chat model classHow to create a custom LLM classCustom RetrieverHow to create toolsHow to debug your LLM appsHow to load CSVsHow to load documents from a directoryHow to load HTMLHow to load JSONHow to load MarkdownHow to load Microsoft Office filesHow to load PDFsHow to create a dynamic (self-constructing) chainText embedding modelsHow to combine results from multiple retrieversHow to select examples from a LangSmith datasetHow to select examples by lengthHow to select examples by maximal marginal relevance (MMR)How to select examples by n-gram overlapHow to select examples by similarityHow to use reference examples when doing extractionHow to handle long text when doing extractionHow to use prompting alone (no tool calling) to do extractionHow to add fallbacks to a runnableHow to filter messagesHybrid SearchHow to use the LangChain indexing APIHow to inspect runnablesLangChain Expression Language CheatsheetHow to cache LLM responsesHow to track token usage for LLMsRun models locallyHow to get log probabilitiesHow to reorder retrieved results to mitigate the \"lost in the middle\" effectHow to split Markdown by HeadersHow to merge consecutive messages of the same typeHow to add message historyHow to migrate from legacy LangChain agents to LangGraphHow to retrieve using multiple vectors per documentHow to pass multimodal data directly to modelsHow to use multimodal promptsHow to create a custom Output ParserHow to use the output-fixing parserHow to parse JSON outputHow to retry when a parsing error occursHow to parse XML outputHow to parse YAML outputHow to use the Parent Document RetrieverHow to use LangChain with different Pydantic versionsHow to add chat historyHow to get a RAG application to add citationsHow to do per-user retrievalHow to get your RAG application to return sourcesHow to stream results from your RAG applicationHow to split JSON dataHow to recursively split text by charactersResponse metadataHow to pass runtime secrets to runnablesHow to do \"self-querying\" retrievalHow to split text based on semantic similarityHow to chain runnablesHow to save and load LangChain objectsHow to split text by tokensHow to do question answering over CSVsHow to deal with large databases when doing SQL question-answeringHow to better prompt when doing SQL question-answeringHow to do query validation as part of SQL question-answeringHow to stream runnablesHow to stream responses from an LLMHow to use a time-weighted vector store retrieverHow to return artifacts from a toolHow to use chat models to call toolsHow to disable parallel tool callingHow to force models to call a toolHow to access the RunnableConfig from a toolHow to pass tool outputs to chat modelsHow to pass run time values to toolsHow to stream events from a toolHow to stream tool callsHow to convert tools to OpenAI FunctionsHow to handle tool errorsHow to use few-shot prompting with tool callingHow to add a human-in-the-loop for toolsHow to bind model-specific toolsHow to trim messagesHow to create and query vector storesConceptual guideEcosystem\\uf8ffü¶ú\\uf8ffüõ†Ô∏è LangSmith\\uf8ffü¶ú\\uf8ffüï∏Ô∏è LangGraphVersionsOverview of v0.2Release policyPydantic compatibilityMigrating to v0.2Migrating to LangChain v0.2astream_events v2ChangesMigrating from v0.0 chainsHow to migrate from v0.0 chainsMigrating from ConstitutionalChainMigrating from ConversationalChainMigrating from ConversationalRetrievalChainMigrating from LLMChainMigrating from LLMMathChainMigrating from LLMRouterChainMigrating from MapReduceDocumentsChainMigrating from MapRerankDocumentsChainMigrating from MultiPromptChainMigrating from RefineDocumentsChainMigrating from RetrievalQAMigrating from StuffDocumentsChainSecurityConceptual guideOn this pageConceptual guideThis section contains introductions to key parts of LangChain.Architecture‚ÄãLangChain as a framework consists of a number of packages.langchain-core‚ÄãThis package contains base abstractions of different components and ways to compose them together.\\nThe interfaces for core components like LLMs, vector stores, retrievers and more are defined here.\\nNo third party integrations are defined here.\\nThe dependencies are kept purposefully very lightweight.Partner packages‚ÄãWhile the long tail of integrations are in langchain-community, we split popular integrations into their own packages (e.g. langchain-openai, langchain-anthropic, etc).\\nThis was done in order to improve support for these important integrations.langchain‚ÄãThe main langchain package contains chains, agents, and retrieval strategies that make up an application\\'s cognitive architecture.\\nThese are NOT third party integrations.\\nAll chains, agents, and retrieval strategies here are NOT specific to any one integration, but rather generic across all integrations.langchain-community‚ÄãThis package contains third party integrations that are maintained by the LangChain community.\\nKey partner packages are separated out (see below).\\nThis contains all integrations for various components (LLMs, vector stores, retrievers).\\nAll dependencies in this package are optional to keep the package as lightweight as possible.langgraph‚Äãlanggraph is an extension of langchain aimed at\\nbuilding robust and stateful multi-actor applications with LLMs by modeling steps as edges and nodes in a graph.LangGraph exposes high level interfaces for creating common types of agents, as well as a low-level API for composing custom flows.langserve‚ÄãA package to deploy LangChain chains as REST APIs. Makes it easy to get a production ready API up and running.LangSmith‚ÄãA developer platform that lets you debug, test, evaluate, and monitor LLM applications.LangChain Expression Language (LCEL)‚ÄãLangChain Expression Language, or LCEL, is a declarative way to chain LangChain components.\\nLCEL was designed from day 1 to support putting prototypes in production, with no code changes, from the simplest ‚Äúprompt + LLM‚Äù chain to the most complex chains (we‚Äôve seen folks successfully run LCEL chains with 100s of steps in production). To highlight a few of the reasons you might want to use LCEL:First-class streaming support\\nWhen you build your chains with LCEL you get the best possible time-to-first-token (time elapsed until the first chunk of output comes out). For some chains this means eg. we stream tokens straight from an LLM to a streaming output parser, and you get back parsed, incremental chunks of output at the same rate as the LLM provider outputs the raw tokens.Async support\\nAny chain built with LCEL can be called both with the synchronous API (eg. in your Jupyter notebook while prototyping) as well as with the asynchronous API (eg. in a LangServe server). This enables using the same code for prototypes and in production, with great performance, and the ability to handle many concurrent requests in the same server.Optimized parallel execution\\nWhenever your LCEL chains have steps that can be executed in parallel (eg if you fetch documents from multiple retrievers) we automatically do it, both in the sync and the async interfaces, for the smallest possible latency.Retries and fallbacks\\nConfigure retries and fallbacks for any part of your LCEL chain. This is a great way to make your chains more reliable at scale. We‚Äôre currently working on adding streaming support for retries/fallbacks, so you can get the added reliability without any latency cost.Access intermediate results\\nFor more complex chains it‚Äôs often very useful to access the results of intermediate steps even before the final output is produced. This can be used to let end-users know something is happening, or even just to debug your chain. You can stream intermediate results, and it‚Äôs available on every LangServe server.Input and output schemas\\nInput and output schemas give every LCEL chain Pydantic and JSONSchema schemas inferred from the structure of your chain. This can be used for validation of inputs and outputs, and is an integral part of LangServe.Seamless LangSmith tracing\\nAs your chains get more and more complex, it becomes increasingly important to understand what exactly is happening at every step.\\nWith LCEL, all steps are automatically logged to LangSmith for maximum observability and debuggability.LCEL aims to provide consistency around behavior and customization over legacy subclassed chains such as LLMChain and\\nConversationalRetrievalChain. Many of these legacy chains hide important details like prompts, and as a wider variety\\nof viable models emerge, customization has become more and more important.If you are currently using one of these legacy chains, please see this guide for guidance on how to migrate.For guides on how to do specific tasks with LCEL, check out the relevant how-to guides.Runnable interface‚ÄãTo make it as easy as possible to create custom chains, we\\'ve implemented a \"Runnable\" protocol. Many LangChain components implement the Runnable protocol, including chat models, LLMs, output parsers, retrievers, prompt templates, and more. There are also several useful primitives for working with runnables, which you can read about below.This is a standard interface, which makes it easy to define custom chains as well as invoke them in a standard way.\\nThe standard interface includes:stream: stream back chunks of the responseinvoke: call the chain on an inputbatch: call the chain on a list of inputsThese also have corresponding async methods that should be used with asyncio await syntax for concurrency:astream: stream back chunks of the response asyncainvoke: call the chain on an input asyncabatch: call the chain on a list of inputs asyncastream_log: stream back intermediate steps as they happen, in addition to the final responseastream_events: beta stream events as they happen in the chain (introduced in langchain-core 0.1.14)The input type and output type varies by component:ComponentInput TypeOutput TypePromptDictionaryPromptValueChatModelSingle string, list of chat messages or a PromptValueChatMessageLLMSingle string, list of chat messages or a PromptValueStringOutputParserThe output of an LLM or ChatModelDepends on the parserRetrieverSingle stringList of DocumentsToolSingle string or dictionary, depending on the toolDepends on the toolAll runnables expose input and output schemas to inspect the inputs and outputs:input_schema: an input Pydantic model auto-generated from the structure of the Runnableoutput_schema: an output Pydantic model auto-generated from the structure of the RunnableComponents‚ÄãLangChain provides standard, extendable interfaces and external integrations for various components useful for building with LLMs.\\nSome components LangChain implements, some components we rely on third-party integrations for, and others are a mix.Chat models‚ÄãLanguage models that use a sequence of messages as inputs and return chat messages as outputs (as opposed to using plain text).\\nThese are traditionally newer models (older models are generally LLMs, see below).\\nChat models support the assignment of distinct roles to conversation messages, helping to distinguish messages from the AI, users, and instructions such as system messages.Although the underlying models are messages in, message out, the LangChain wrappers also allow these models to take a string as input. This means you can easily use chat models in place of LLMs.When a string is passed in as input, it is converted to a HumanMessage and then passed to the underlying model.LangChain does not host any Chat Models, rather we rely on third party integrations.We have some standardized parameters when constructing ChatModels:model: the name of the modeltemperature: the sampling temperaturetimeout: request timeoutmax_tokens: max tokens to generatestop: default stop sequencesmax_retries: max number of times to retry requestsapi_key: API key for the model providerbase_url: endpoint to send requests toSome important things to note:standard params only apply to model providers that expose parameters with the intended functionality. For example, some providers do not expose a configuration for maximum output tokens, so max_tokens can\\'t be supported on these.standard params are currently only enforced on integrations that have their own integration packages (e.g. langchain-openai, langchain-anthropic, etc.), they\\'re not enforced on models in langchain-community.ChatModels also accept other parameters that are specific to that integration. To find all the parameters supported by a ChatModel head to the API reference for that model.infoSome chat models have been fine-tuned for tool calling and provide a dedicated API for it.\\nGenerally, such models are better at tool calling than non-fine-tuned models, and are recommended for use cases that require tool calling.\\nPlease see the tool calling section for more information.For specifics on how to use chat models, see the relevant how-to guides here.Multimodality‚ÄãSome chat models are multimodal, accepting images, audio and even video as inputs. These are still less common, meaning model providers haven\\'t standardized on the \"best\" way to define the API. Multimodal outputs are even less common. As such, we\\'ve kept our multimodal abstractions fairly light weight and plan to further solidify the multimodal APIs and interaction patterns as the field matures.In LangChain, most chat models that support multimodal inputs also accept those values in OpenAI\\'s content blocks format. So far this is restricted to image inputs. For models like Gemini which support video and other bytes input, the APIs also support the native, model-specific representations.For specifics on how to use multimodal models, see the relevant how-to guides here.For a full list of LangChain model providers with multimodal models, check out this table.LLMs‚ÄãcautionPure text-in/text-out LLMs tend to be older or lower-level. Many popular models are best used as chat completion models,\\neven for non-chat use cases.You are probably looking for the section above instead.Language models that takes a string as input and returns a string.\\nThese are traditionally older models (newer models generally are Chat Models, see above).Although the underlying models are string in, string out, the LangChain wrappers also allow these models to take messages as input.\\nThis gives them the same interface as Chat Models.\\nWhen messages are passed in as input, they will be formatted into a string under the hood before being passed to the underlying model.LangChain does not host any LLMs, rather we rely on third party integrations.For specifics on how to use LLMs, see the relevant how-to guides here.Messages‚ÄãSome language models take a list of messages as input and return a message.\\nThere are a few different types of messages.\\nAll messages have a role, content, and response_metadata property.The role describes WHO is saying the message. The standard roles are \"user\", \"assistant\", \"system\", and \"tool\".\\nLangChain has different message classes for different roles.The content property describes the content of the message.\\nThis can be a few different things:A string (most models deal this type of content)A List of dictionaries (this is used for multimodal input, where the dictionary contains information about that input type and that input location)Optionally, messages can have a name property which allows for differentiating between multiple speakers with the same role.\\nFor example, if there are two users in the chat history it can be useful to differentiate between them. Not all models support this.HumanMessage‚ÄãThis represents a message with role \"user\".AIMessage‚ÄãThis represents a message with role \"assistant\". In addition to the content property, these messages also have:response_metadataThe response_metadata property contains additional metadata about the response. The data here is often specific to each model provider.\\nThis is where information like log-probs and token usage may be stored.tool_callsThese represent a decision from an language model to call a tool. They are included as part of an AIMessage output.\\nThey can be accessed from there with the .tool_calls property.This property returns a list of ToolCalls. A ToolCall is a dictionary with the following arguments:name: The name of the tool that should be called.args: The arguments to that tool.id: The id of that tool call.SystemMessage‚ÄãThis represents a message with role \"system\", which tells the model how to behave. Not every model provider supports this.ToolMessage‚ÄãThis represents a message with role \"tool\", which contains the result of calling a tool. In addition to role and content, this message has:a tool_call_id field which conveys the id of the call to the tool that was called to produce this result.an artifact field which can be used to pass along arbitrary artifacts of the tool execution which are useful to track but which should not be sent to the model.(Legacy) FunctionMessage‚ÄãThis is a legacy message type, corresponding to OpenAI\\'s legacy function-calling API. ToolMessage should be used instead to correspond to the updated tool-calling API.This represents the result of a function call. In addition to role and content, this message has a name parameter which conveys the name of the function that was called to produce this result.Prompt templates‚ÄãPrompt templates help to translate user input and parameters into instructions for a language model.\\nThis can be used to guide a model\\'s response, helping it understand the context and generate relevant and coherent language-based output.Prompt Templates take as input a dictionary, where each key represents a variable in the prompt template to fill in.Prompt Templates output a PromptValue. This PromptValue can be passed to an LLM or a ChatModel, and can also be cast to a string or a list of messages.\\nThe reason this PromptValue exists is to make it easy to switch between strings and messages.There are a few different types of prompt templates:String PromptTemplates‚ÄãThese prompt templates are used to format a single string, and generally are used for simpler inputs.\\nFor example, a common way to construct and use a PromptTemplate is as follows:from langchain_core.prompts import PromptTemplateprompt_template = PromptTemplate.from_template(\"Tell me a joke about {topic}\")prompt_template.invoke({\"topic\": \"cats\"})API Reference:PromptTemplateChatPromptTemplates‚ÄãThese prompt templates are used to format a list of messages. These \"templates\" consist of a list of templates themselves.\\nFor example, a common way to construct and use a ChatPromptTemplate is as follows:from langchain_core.prompts import ChatPromptTemplateprompt_template = ChatPromptTemplate.from_messages([    (\"system\", \"You are a helpful assistant\"),    (\"user\", \"Tell me a joke about {topic}\")])prompt_template.invoke({\"topic\": \"cats\"})API Reference:ChatPromptTemplateIn the above example, this ChatPromptTemplate will construct two messages when called.\\nThe first is a system message, that has no variables to format.\\nThe second is a HumanMessage, and will be formatted by the topic variable the user passes in.MessagesPlaceholder‚ÄãThis prompt template is responsible for adding a list of messages in a particular place.\\nIn the above ChatPromptTemplate, we saw how we could format two messages, each one a string.\\nBut what if we wanted the user to pass in a list of messages that we would slot into a particular spot?\\nThis is how you use MessagesPlaceholder.from langchain_core.prompts import ChatPromptTemplate, MessagesPlaceholderfrom langchain_core.messages import HumanMessageprompt_template = ChatPromptTemplate.from_messages([    (\"system\", \"You are a helpful assistant\"),    MessagesPlaceholder(\"msgs\")])prompt_template.invoke({\"msgs\": [HumanMessage(content=\"hi!\")]})API Reference:ChatPromptTemplate | MessagesPlaceholder | HumanMessageThis will produce a list of two messages, the first one being a system message, and the second one being the HumanMessage we passed in.\\nIf we had passed in 5 messages, then it would have produced 6 messages in total (the system message plus the 5 passed in).\\nThis is useful for letting a list of messages be slotted into a particular spot.An alternative way to accomplish the same thing without using the MessagesPlaceholder class explicitly is:prompt_template = ChatPromptTemplate.from_messages([    (\"system\", \"You are a helpful assistant\"),    (\"placeholder\", \"{msgs}\") # <-- This is the changed part])For specifics on how to use prompt templates, see the relevant how-to guides here.Example selectors‚ÄãOne common prompting technique for achieving better performance is to include examples as part of the prompt.\\nThis is known as few-shot prompting.\\nThis gives the language model concrete examples of how it should behave.\\nSometimes these examples are hardcoded into the prompt, but for more advanced situations it may be nice to dynamically select them.\\nExample Selectors are classes responsible for selecting and then formatting examples into prompts.For specifics on how to use example selectors, see the relevant how-to guides here.Output parsers‚ÄãnoteThe information here refers to parsers that take a text output from a model try to parse it into a more structured representation.\\nMore and more models are supporting function (or tool) calling, which handles this automatically.\\nIt is recommended to use function/tool calling rather than output parsing.\\nSee documentation for that here.Responsible for taking the output of a model and transforming it to a more suitable format for downstream tasks.\\nUseful when you are using LLMs to generate structured data, or to normalize output from chat models and LLMs.LangChain has lots of different types of output parsers. This is a list of output parsers LangChain supports. The table below has various pieces of information:Name: The name of the output parserSupports Streaming: Whether the output parser supports streaming.Has Format Instructions: Whether the output parser has format instructions. This is generally available except when (a) the desired schema is not specified in the prompt but rather in other parameters (like OpenAI function calling), or (b) when the OutputParser wraps another OutputParser.Calls LLM: Whether this output parser itself calls an LLM. This is usually only done by output parsers that attempt to correct misformatted output.Input Type: Expected input type. Most output parsers work on both strings and messages, but some (like OpenAI Functions) need a message with specific kwargs.Output Type: The output type of the object returned by the parser.Description: Our commentary on this output parser and when to use it.NameSupports StreamingHas Format InstructionsCalls LLMInput TypeOutput TypeDescriptionJSON‚úÖ‚úÖstr | MessageJSON objectReturns a JSON object as specified. You can specify a Pydantic model and it will return JSON for that model. Probably the most reliable output parser for getting structured data that does NOT use function calling.XML‚úÖ‚úÖstr | MessagedictReturns a dictionary of tags. Use when XML output is needed. Use with models that are good at writing XML (like Anthropic\\'s).CSV‚úÖ‚úÖstr | MessageList[str]Returns a list of comma separated values.OutputFixing‚úÖstr | MessageWraps another output parser. If that output parser errors, then this will pass the error message and the bad output to an LLM and ask it to fix the output.RetryWithError‚úÖstr | MessageWraps another output parser. If that output parser errors, then this will pass the original inputs, the bad output, and the error message to an LLM and ask it to fix it. Compared to OutputFixingParser, this one also sends the original instructions.Pydantic‚úÖstr | Messagepydantic.BaseModelTakes a user defined Pydantic model and returns data in that format.YAML‚úÖstr | Messagepydantic.BaseModelTakes a user defined Pydantic model and returns data in that format. Uses YAML to encode it.PandasDataFrame‚úÖstr | MessagedictUseful for doing operations with pandas DataFrames.Enum‚úÖstr | MessageEnumParses response into one of the provided enum values.Datetime‚úÖstr | Messagedatetime.datetimeParses response into a datetime string.Structured‚úÖstr | MessageDict[str, str]An output parser that returns structured information. It is less powerful than other output parsers since it only allows for fields to be strings. This can be useful when you are working with smaller LLMs.For specifics on how to use output parsers, see the relevant how-to guides here.Chat history‚ÄãMost LLM applications have a conversational interface.\\nAn essential component of a conversation is being able to refer to information introduced earlier in the conversation.\\nAt bare minimum, a conversational system should be able to access some window of past messages directly.The concept of ChatHistory refers to a class in LangChain which can be used to wrap an arbitrary chain.\\nThis ChatHistory will keep track of inputs and outputs of the underlying chain, and append them as messages to a message database.\\nFuture interactions will then load those messages and pass them into the chain as part of the input.Documents‚ÄãA Document object in LangChain contains information about some data. It has two attributes:page_content: str: The content of this document. Currently is only a string.metadata: dict: Arbitrary metadata associated with this document. Can track the document id, file name, etc.Document loaders‚ÄãThese classes load Document objects. LangChain has hundreds of integrations with various data sources to load data from: Slack, Notion, Google Drive, etc.Each DocumentLoader has its own specific parameters, but they can all be invoked in the same way with the .load method.\\nAn example use case is as follows:from langchain_community.document_loaders.csv_loader import CSVLoaderloader = CSVLoader(    ...  # <-- Integration specific parameters here)data = loader.load()API Reference:CSVLoaderFor specifics on how to use document loaders, see the relevant how-to guides here.Text splitters‚ÄãOnce you\\'ve loaded documents, you\\'ll often want to transform them to better suit your application. The simplest example is you may want to split a long document into smaller chunks that can fit into your model\\'s context window. LangChain has a number of built-in document transformers that make it easy to split, combine, filter, and otherwise manipulate documents.When you want to deal with long pieces of text, it is necessary to split up that text into chunks. As simple as this sounds, there is a lot of potential complexity here. Ideally, you want to keep the semantically related pieces of text together. What \"semantically related\" means could depend on the type of text. This notebook showcases several ways to do that.At a high level, text splitters work as following:Split the text up into small, semantically meaningful chunks (often sentences).Start combining these small chunks into a larger chunk until you reach a certain size (as measured by some function).Once you reach that size, make that chunk its own piece of text and then start creating a new chunk of text with some overlap (to keep context between chunks).That means there are two different axes along which you can customize your text splitter:How the text is splitHow the chunk size is measuredFor specifics on how to use text splitters, see the relevant how-to guides here.Embedding models‚ÄãEmbedding models create a vector representation of a piece of text. You can think of a vector as an array of numbers that captures the semantic meaning of the text.\\nBy representing the text in this way, you can perform mathematical operations that allow you to do things like search for other pieces of text that are most similar in meaning.\\nThese natural language search capabilities underpin many types of context retrieval,\\nwhere we provide an LLM with the relevant data it needs to effectively respond to a query.The Embeddings class is a class designed for interfacing with text embedding models. There are many different embedding model providers (OpenAI, Cohere, Hugging Face, etc) and local models, and this class is designed to provide a standard interface for all of them.The base Embeddings class in LangChain provides two methods: one for embedding documents and one for embedding a query. The former takes as input multiple texts, while the latter takes a single text. The reason for having these as two separate methods is that some embedding providers have different embedding methods for documents (to be searched over) vs queries (the search query itself).For specifics on how to use embedding models, see the relevant how-to guides here.Vector stores‚ÄãOne of the most common ways to store and search over unstructured data is to embed it and store the resulting embedding vectors,\\nand then at query time to embed the unstructured query and retrieve the embedding vectors that are \\'most similar\\' to the embedded query.\\nA vector store takes care of storing embedded data and performing vector search for you.Most vector stores can also store metadata about embedded vectors and support filtering on that metadata before\\nsimilarity search, allowing you more control over returned documents.Vector stores can be converted to the retriever interface by doing:vectorstore = MyVectorStore()retriever = vectorstore.as_retriever()For specifics on how to use vector stores, see the relevant how-to guides here.Retrievers‚ÄãA retriever is an interface that returns documents given an unstructured query.\\nIt is more general than a vector store.\\nA retriever does not need to be able to store documents, only to return (or retrieve) them.\\nRetrievers can be created from vector stores, but are also broad enough to include Wikipedia search and Amazon Kendra.Retrievers accept a string query as input and return a list of Document\\'s as output.For specifics on how to use retrievers, see the relevant how-to guides here.Key-value stores‚ÄãFor some techniques, such as indexing and retrieval with multiple vectors per document or\\ncaching embeddings, having a form of key-value (KV) storage is helpful.LangChain includes a BaseStore interface,\\nwhich allows for storage of arbitrary data. However, LangChain components that require KV-storage accept a\\nmore specific BaseStore[str, bytes] instance that stores binary data (referred to as a ByteStore), and internally take care of\\nencoding and decoding data for their specific needs.This means that as a user, you only need to think about one type of store rather than different ones for different types of data.Interface‚ÄãAll BaseStores support the following interface. Note that the interface allows\\nfor modifying multiple key-value pairs at once:mget(key: Sequence[str]) -> List[Optional[bytes]]: get the contents of multiple keys, returning None if the key does not existmset(key_value_pairs: Sequence[Tuple[str, bytes]]) -> None: set the contents of multiple keysmdelete(key: Sequence[str]) -> None: delete multiple keysyield_keys(prefix: Optional[str] = None) -> Iterator[str]: yield all keys in the store, optionally filtering by a prefixFor key-value store implementations, see this section.Tools‚ÄãTools are utilities designed to be called by a model: their inputs are designed to be generated by models, and their outputs are designed to be passed back to models.\\nTools are needed whenever you want a model to control parts of your code or call out to external APIs.A tool consists of:The name of the tool.A description of what the tool does.A JSON schema defining the inputs to the tool.A function (and, optionally, an async variant of the function).When a tool is bound to a model, the name, description and JSON schema are provided as context to the model.\\nGiven a list of tools and a set of instructions, a model can request to call one or more tools with specific inputs.\\nTypical usage may look like the following:tools = [...] # Define a list of toolsllm_with_tools = llm.bind_tools(tools)ai_msg = llm_with_tools.invoke(\"do xyz...\")# -> AIMessage(tool_calls=[ToolCall(...), ...], ...)The AIMessage returned from the model MAY have tool_calls associated with it.\\nRead this guide for more information on what the response type may look like.Once the chosen tools are invoked, the results can be passed back to the model so that it can complete whatever task\\nit\\'s performing.\\nThere are generally two different ways to invoke the tool and pass back the response:Invoke with just the arguments‚ÄãWhen you invoke a tool with just the arguments, you will get back the raw tool output (usually a string).\\nThis generally looks like:# You will want to previously check that the LLM returned tool callstool_call = ai_msg.tool_calls[0]# ToolCall(args={...}, id=..., ...)tool_output = tool.invoke(tool_call[\"args\"])tool_message = ToolMessage(    content=tool_output,    tool_call_id=tool_call[\"id\"],    name=tool_call[\"name\"])Note that the content field will generally be passed back to the model.\\nIf you do not want the raw tool response to be passed to the model, but you still want to keep it around,\\nyou can transform the tool output but also pass it as an artifact (read more about ToolMessage.artifact here)... # Same code as aboveresponse_for_llm = transform(response)tool_message = ToolMessage(    content=response_for_llm,    tool_call_id=tool_call[\"id\"],    name=tool_call[\"name\"],    artifact=tool_output)Invoke with ToolCall‚ÄãThe other way to invoke a tool is to call it with the full ToolCall that was generated by the model.\\nWhen you do this, the tool will return a ToolMessage.\\nThe benefits of this are that you don\\'t have to write the logic yourself to transform the tool output into a ToolMessage.\\nThis generally looks like:tool_call = ai_msg.tool_calls[0]# -> ToolCall(args={...}, id=..., ...)tool_message = tool.invoke(tool_call)# -> ToolMessage(    content=\"tool result foobar...\",    tool_call_id=...,    name=\"tool_name\")If you are invoking the tool this way and want to include an artifact for the ToolMessage, you will need to have the tool return two things.\\nRead more about defining tools that return artifacts here.Best practices‚ÄãWhen designing tools to be used by a model, it is important to keep in mind that:Chat models that have explicit tool-calling APIs will be better at tool calling than non-fine-tuned models.Models will perform better if the tools have well-chosen names, descriptions, and JSON schemas. This another form of prompt engineering.Simple, narrowly scoped tools are easier for models to use than complex tools.Related‚ÄãFor specifics on how to use tools, see the tools how-to guides.To use a pre-built tool, see the tool integration docs.Toolkits‚ÄãToolkits are collections of tools that are designed to be used together for specific tasks. They have convenient loading methods.All Toolkits expose a get_tools method which returns a list of tools.\\nYou can therefore do:# Initialize a toolkittoolkit = ExampleTookit(...)# Get list of toolstools = toolkit.get_tools()Agents‚ÄãBy themselves, language models can\\'t take actions - they just output text.\\nA big use case for LangChain is creating agents.\\nAgents are systems that use an LLM as a reasoning engine to determine which actions to take and what the inputs to those actions should be.\\nThe results of those actions can then be fed back into the agent and it determine whether more actions are needed, or whether it is okay to finish.LangGraph is an extension of LangChain specifically aimed at creating highly controllable and customizable agents.\\nPlease check out that documentation for a more in depth overview of agent concepts.There is a legacy agent concept in LangChain that we are moving towards deprecating: AgentExecutor.\\nAgentExecutor was essentially a runtime for agents.\\nIt was a great place to get started, however, it was not flexible enough as you started to have more customized agents.\\nIn order to solve that we built LangGraph to be this flexible, highly-controllable runtime.If you are still using AgentExecutor, do not fear: we still have a guide on how to use AgentExecutor.\\nIt is recommended, however, that you start to transition to LangGraph.\\nIn order to assist in this we have put together a transition guide on how to do so.ReAct agents‚ÄãOne popular architecture for building agents is ReAct.\\nReAct combines reasoning and acting in an iterative process - in fact the name \"ReAct\" stands for \"Reason\" and \"Act\".The general flow looks like this:The model will \"think\" about what step to take in response to an input and any previous observations.The model will then choose an action from available tools (or choose to respond to the user).The model will generate arguments to that tool.The agent runtime (executor) will parse out the chosen tool and call it with the generated arguments.The executor will return the results of the tool call back to the model as an observation.This process repeats until the agent chooses to respond.There are general prompting based implementations that do not require any model-specific features, but the most\\nreliable implementations use features like tool calling to reliably format outputs\\nand reduce variance.Please see the LangGraph documentation for more information,\\nor this how-to guide for specific information on migrating to LangGraph.Callbacks‚ÄãLangChain provides a callbacks system that allows you to hook into the various stages of your LLM application. This is useful for logging, monitoring, streaming, and other tasks.You can subscribe to these events by using the callbacks argument available throughout the API. This argument is list of handler objects, which are expected to implement one or more of the methods described below in more detail.Callback Events‚ÄãEventEvent TriggerAssociated MethodChat model startWhen a chat model startson_chat_model_startLLM startWhen a llm startson_llm_startLLM new tokenWhen an llm OR chat model emits a new tokenon_llm_new_tokenLLM endsWhen an llm OR chat model endson_llm_endLLM errorsWhen an llm OR chat model errorson_llm_errorChain startWhen a chain starts runningon_chain_startChain endWhen a chain endson_chain_endChain errorWhen a chain errorson_chain_errorTool startWhen a tool starts runningon_tool_startTool endWhen a tool endson_tool_endTool errorWhen a tool errorson_tool_errorAgent actionWhen an agent takes an actionon_agent_actionAgent finishWhen an agent endson_agent_finishRetriever startWhen a retriever startson_retriever_startRetriever endWhen a retriever endson_retriever_endRetriever errorWhen a retriever errorson_retriever_errorTextWhen arbitrary text is runon_textRetryWhen a retry event is runon_retryCallback handlers‚ÄãCallback handlers can either be sync or async:Sync callback handlers implement the BaseCallbackHandler interface.Async callback handlers implement the AsyncCallbackHandler interface.During run-time LangChain configures an appropriate callback manager (e.g., CallbackManager or AsyncCallbackManager which will be responsible for calling the appropriate method on each \"registered\" callback handler when the event is triggered.Passing callbacks‚ÄãThe callbacks property is available on most objects throughout the API (Models, Tools, Agents, etc.) in two different places:The callbacks are available on most objects throughout the API (Models, Tools, Agents, etc.) in two different places:Request time callbacks: Passed at the time of the request in addition to the input data.\\nAvailable on all standard Runnable objects. These callbacks are INHERITED by all children\\nof the object they are defined on. For example, chain.invoke({\"number\": 25}, {\"callbacks\": [handler]}).Constructor callbacks: chain = TheNameOfSomeChain(callbacks=[handler]). These callbacks\\nare passed as arguments to the constructor of the object. The callbacks are scoped\\nonly to the object they are defined on, and are not inherited by any children of the object.dangerConstructor callbacks are scoped only to the object they are defined on. They are not inherited by children\\nof the object.If you\\'re creating a custom chain or runnable, you need to remember to propagate request time\\ncallbacks to any child objects.Async in Python<=3.10Any RunnableLambda, a RunnableGenerator, or Tool that invokes other runnables\\nand is running async in python<=3.10, will have to propagate callbacks to child\\nobjects manually. This is because LangChain cannot automatically propagate\\ncallbacks to child objects in this case.This is a common reason why you may fail to see events being emitted from custom\\nrunnables or tools.For specifics on how to use callbacks, see the relevant how-to guides here.Techniques‚ÄãStreaming‚ÄãIndividual LLM calls often run for much longer than traditional resource requests.\\nThis compounds when you build more complex chains or agents that require multiple reasoning steps.Fortunately, LLMs generate output iteratively, which means it\\'s possible to show sensible intermediate results\\nbefore the final response is ready. Consuming output as soon as it becomes available has therefore become a vital part of the UX\\naround building apps with LLMs to help alleviate latency issues, and LangChain aims to have first-class support for streaming.Below, we\\'ll discuss some concepts and considerations around streaming in LangChain..stream() and .astream()‚ÄãMost modules in LangChain include the .stream() method (and the equivalent .astream() method for async environments) as an ergonomic streaming interface.\\n.stream() returns an iterator, which you can consume with a simple for loop. Here\\'s an example with a chat model:from langchain_anthropic import ChatAnthropicmodel = ChatAnthropic(model=\"claude-3-sonnet-20240229\")for chunk in model.stream(\"what color is the sky?\"):    print(chunk.content, end=\"|\", flush=True)API Reference:ChatAnthropicFor models (or other components) that don\\'t support streaming natively, this iterator would just yield a single chunk, but\\nyou could still use the same general pattern when calling them. Using .stream() will also automatically call the model in streaming mode\\nwithout the need to provide additional config.The type of each outputted chunk depends on the type of component - for example, chat models yield AIMessageChunks.\\nBecause this method is part of LangChain Expression Language,\\nyou can handle formatting differences from different outputs using an output parser to transform\\neach yielded chunk.You can check out this guide for more detail on how to use .stream()..astream_events()‚ÄãWhile the .stream() method is intuitive, it can only return the final generated value of your chain. This is fine for single LLM calls,\\nbut as you build more complex chains of several LLM calls together, you may want to use the intermediate values of\\nthe chain alongside the final output - for example, returning sources alongside the final generation when building a chat\\nover documents app.There are ways to do this using callbacks, or by constructing your chain in such a way that it passes intermediate\\nvalues to the end with something like chained .assign() calls, but LangChain also includes an\\n.astream_events() method that combines the flexibility of callbacks with the ergonomics of .stream(). When called, it returns an iterator\\nwhich yields various types of events that you can filter and process according\\nto the needs of your project.Here\\'s one small example that prints just events containing streamed chat model output:from langchain_core.output_parsers import StrOutputParserfrom langchain_core.prompts import ChatPromptTemplatefrom langchain_anthropic import ChatAnthropicmodel = ChatAnthropic(model=\"claude-3-sonnet-20240229\")prompt = ChatPromptTemplate.from_template(\"tell me a joke about {topic}\")parser = StrOutputParser()chain = prompt | model | parserasync for event in chain.astream_events({\"topic\": \"parrot\"}, version=\"v2\"):    kind = event[\"event\"]    if kind == \"on_chat_model_stream\":        print(event, end=\"|\", flush=True)API Reference:StrOutputParser | ChatPromptTemplate | ChatAnthropicYou can roughly think of it as an iterator over callback events (though the format differs) - and you can use it on almost all LangChain components!See this guide for more detailed information on how to use .astream_events(),\\nincluding a table listing available events.Callbacks‚ÄãThe lowest level way to stream outputs from LLMs in LangChain is via the callbacks system. You can pass a\\ncallback handler that handles the on_llm_new_token event into LangChain components. When that component is invoked, any\\nLLM or chat model contained in the component calls\\nthe callback with the generated token. Within the callback, you could pipe the tokens into some other destination, e.g. a HTTP response.\\nYou can also handle the on_llm_end event to perform any necessary cleanup.You can see this how-to section for more specifics on using callbacks.Callbacks were the first technique for streaming introduced in LangChain. While powerful and generalizable,\\nthey can be unwieldy for developers. For example:You need to explicitly initialize and manage some aggregator or other stream to collect results.The execution order isn\\'t explicitly guaranteed, and you could theoretically have a callback run after the .invoke() method finishes.Providers would often make you pass an additional parameter to stream outputs instead of returning them all at once.You would often ignore the result of the actual model call in favor of callback results.Tokens‚ÄãThe unit that most model providers use to measure input and output is via a unit called a token.\\nTokens are the basic units that language models read and generate when processing or producing text.\\nThe exact definition of a token can vary depending on the specific way the model was trained -\\nfor instance, in English, a token could be a single word like \"apple\", or a part of a word like \"app\".When you send a model a prompt, the words and characters in the prompt are encoded into tokens using a tokenizer.\\nThe model then streams back generated output tokens, which the tokenizer decodes into human-readable text.\\nThe below example shows how OpenAI models tokenize LangChain is cool!:You can see that it gets split into 5 different tokens, and that the boundaries between tokens are not exactly the same as word boundaries.The reason language models use tokens rather than something more immediately intuitive like \"characters\"\\nhas to do with how they process and understand text. At a high-level, language models iteratively predict their next generated output based on\\nthe initial input and their previous generations. Training the model using tokens language models to handle linguistic\\nunits (like words or subwords) that carry meaning, rather than individual characters, which makes it easier for the model\\nto learn and understand the structure of the language, including grammar and context.\\nFurthermore, using tokens can also improve efficiency, since the model processes fewer units of text compared to character-level processing.Function/tool calling‚ÄãinfoWe use the term tool calling interchangeably with function calling. Although\\nfunction calling is sometimes meant to refer to invocations of a single function,\\nwe treat all models as though they can return multiple tool or function calls in\\neach message.Tool calling allows a chat model to respond to a given prompt by generating output that\\nmatches a user-defined schema.While the name implies that the model is performing\\nsome action, this is actually not the case! The model only generates the arguments to a tool, and actually running the tool (or not) is up to the user.\\nOne common example where you wouldn\\'t want to call a function with the generated arguments\\nis if you want to extract structured output matching some schema\\nfrom unstructured text. You would give the model an \"extraction\" tool that takes\\nparameters matching the desired schema, then treat the generated output as your final\\nresult.Tool calling is not universal, but is supported by many popular LLM providers, including Anthropic,\\nCohere, Google,\\nMistral, OpenAI, and even for locally-running models via Ollama.LangChain provides a standardized interface for tool calling that is consistent across different models.The standard interface consists of:ChatModel.bind_tools(): a method for specifying which tools are available for a model to call. This method accepts LangChain tools as well as Pydantic objects.AIMessage.tool_calls: an attribute on the AIMessage returned from the model for accessing the tool calls requested by the model.Tool usage‚ÄãAfter the model calls tools, you can use the tool by invoking it, then passing the arguments back to the model.\\nLangChain provides the Tool abstraction to help you handle this.The general flow is this:Generate tool calls with a chat model in response to a query.Invoke the appropriate tools using the generated tool call as arguments.Format the result of the tool invocations as ToolMessages.Pass the entire list of messages back to the model so that it can generate a final answer (or call more tools).This is how tool calling agents perform tasks and answer queries.Check out some more focused guides below:How to use chat models to call toolsHow to pass tool outputs to chat modelsBuilding an agent with LangGraphStructured output‚ÄãLLMs are capable of generating arbitrary text. This enables the model to respond appropriately to a wide\\nrange of inputs, but for some use-cases, it can be useful to constrain the LLM\\'s output\\nto a specific format or structure. This is referred to as structured output.For example, if the output is to be stored in a relational database,\\nit is much easier if the model generates output that adheres to a defined schema or format.\\nExtracting specific information from unstructured text is another\\ncase where this is particularly useful. Most commonly, the output format will be JSON,\\nthough other formats such as YAML can be useful too. Below, we\\'ll discuss\\na few ways to get structured output from models in LangChain..with_structured_output()‚ÄãFor convenience, some LangChain chat models support a .with_structured_output()\\nmethod. This method only requires a schema as input, and returns a dict or Pydantic object.\\nGenerally, this method is only present on models that support one of the more advanced methods described below,\\nand will use one of them under the hood. It takes care of importing a suitable output parser and\\nformatting the schema in the right format for the model.Here\\'s an example:from typing import Optionalfrom langchain_core.pydantic_v1 import BaseModel, Fieldclass Joke(BaseModel):    \"\"\"Joke to tell user.\"\"\"    setup: str = Field(description=\"The setup of the joke\")    punchline: str = Field(description=\"The punchline to the joke\")    rating: Optional[int] = Field(description=\"How funny the joke is, from 1 to 10\")structured_llm = llm.with_structured_output(Joke)structured_llm.invoke(\"Tell me a joke about cats\")Joke(setup=\\'Why was the cat sitting on the computer?\\', punchline=\\'To keep an eye on the mouse!\\', rating=None)We recommend this method as a starting point when working with structured output:It uses other model-specific features under the hood, without the need to import an output parser.For the models that use tool calling, no special prompting is needed.If multiple underlying techniques are supported, you can supply a method parameter to\\ntoggle which one is used.You may want or need to use other techniques if:The chat model you are using does not support tool calling.You are working with very complex schemas and the model is having trouble generating outputs that conform.For more information, check out this how-to guide.You can also check out this table for a list of models that support\\nwith_structured_output().Raw prompting‚ÄãThe most intuitive way to get a model to structure output is to ask nicely.\\nIn addition to your query, you can give instructions describing what kind of output you\\'d like, then\\nparse the output using an output parser to convert the raw\\nmodel message or string output into something more easily manipulated.The biggest benefit to raw prompting is its flexibility:Raw prompting does not require any special model features, only sufficient reasoning capability to understand\\nthe passed schema.You can prompt for any format you\\'d like, not just JSON. This can be useful if the model you\\nare using is more heavily trained on a certain type of data, such as XML or YAML.However, there are some drawbacks too:LLMs are non-deterministic, and prompting a LLM to consistently output data in the exactly correct format\\nfor smooth parsing can be surprisingly difficult and model-specific.Individual models have quirks depending on the data they were trained on, and optimizing prompts can be quite difficult.\\nSome may be better at interpreting JSON schema, others may be best with TypeScript definitions,\\nand still others may prefer XML.While features offered by model providers may increase reliability, prompting techniques remain important for tuning your\\nresults no matter which method you choose.JSON mode‚ÄãSome models, such as Mistral, OpenAI,\\nTogether AI and Ollama,\\nsupport a feature called JSON mode, usually enabled via config.When enabled, JSON mode will constrain the model\\'s output to always be some sort of valid JSON.\\nOften they require some custom prompting, but it\\'s usually much less burdensome than completely raw prompting and\\nmore along the lines of, \"you must always return JSON\". The output also generally easier to parse.It\\'s also generally simpler to use directly and more commonly available than tool calling, and can give\\nmore flexibility around prompting and shaping results than tool calling.Here\\'s an example:from langchain_core.prompts import ChatPromptTemplatefrom langchain_openai import ChatOpenAIfrom langchain.output_parsers.json import SimpleJsonOutputParsermodel = ChatOpenAI(    model=\"gpt-4o\",    model_kwargs={ \"response_format\": { \"type\": \"json_object\" } },)prompt = ChatPromptTemplate.from_template(    \"Answer the user\\'s question to the best of your ability.\"    \\'You must always output a JSON object with an \"answer\" key and a \"followup_question\" key.\\'    \"{question}\")chain = prompt | model | SimpleJsonOutputParser()chain.invoke({ \"question\": \"What is the powerhouse of the cell?\" })API Reference:ChatPromptTemplate | ChatOpenAI | SimpleJsonOutputParser{\\'answer\\': \\'The powerhouse of the cell is the mitochondrion. It is responsible for producing energy in the form of ATP through cellular respiration.\\', \\'followup_question\\': \\'Would you like to know more about how mitochondria produce energy?\\'}For a full list of model providers that support JSON mode, see this table.Tool calling‚ÄãFor models that support it, tool calling can be very convenient for structured output. It removes the\\nguesswork around how best to prompt schemas in favor of a built-in model feature.It works by first binding the desired schema either directly or via a LangChain tool to a\\nchat model using the .bind_tools() method. The model will then generate an AIMessage containing\\na tool_calls field containing args that match the desired shape.There are several acceptable formats you can use to bind tools to a model in LangChain. Here\\'s one example:from langchain_core.pydantic_v1 import BaseModel, Fieldfrom langchain_openai import ChatOpenAIclass ResponseFormatter(BaseModel):    \"\"\"Always use this tool to structure your response to the user.\"\"\"    answer: str = Field(description=\"The answer to the user\\'s question\")    followup_question: str = Field(description=\"A followup question the user could ask\")model = ChatOpenAI(    model=\"gpt-4o\",    temperature=0,)model_with_tools = model.bind_tools([ResponseFormatter])ai_msg = model_with_tools.invoke(\"What is the powerhouse of the cell?\")ai_msg.tool_calls[0][\"args\"]API Reference:ChatOpenAI{\\'answer\\': \"The powerhouse of the cell is the mitochondrion. It generates most of the cell\\'s supply of adenosine triphosphate (ATP), which is used as a source of chemical energy.\", \\'followup_question\\': \\'How do mitochondria generate ATP?\\'}Tool calling is a generally consistent way to get a model to generate structured output, and is the default technique\\nused for the .with_structured_output() method when a model supports it.The following how-to guides are good practical resources for using function/tool calling for structured output:How to return structured data from an LLMHow to use a model to call toolsFor a full list of model providers that support tool calling, see this table.Few-shot prompting‚ÄãOne of the most effective ways to improve model performance is to give a model examples of what you want it to do. The technique of adding example inputs and expected outputs to a model prompt is known as \"few-shot prompting\". There are a few things to think about when doing few-shot prompting:How are examples generated?How many examples are in each prompt?How are examples selected at runtime?How are examples formatted in the prompt?Here are the considerations for each.1. Generating examples‚ÄãThe first and most important step of few-shot prompting is coming up with a good dataset of examples. Good examples should be relevant at runtime, clear, informative, and provide information that was not already known to the model.At a high-level, the basic ways to generate examples are:Manual: a person/people generates examples they think are useful.Better model: a better (presumably more expensive/slower) model\\'s responses are used as examples for a worse (presumably cheaper/faster) model.User feedback: users (or labelers) leave feedback on interactions with the application and examples are generated based on that feedback (for example, all interactions with positive feedback could be turned into examples).LLM feedback: same as user feedback but the process is automated by having models evaluate themselves.Which approach is best depends on your task. For tasks where a small number core principles need to be understood really well, it can be valuable hand-craft a few really good examples.\\nFor tasks where the space of correct behaviors is broader and more nuanced, it can be useful to generate many examples in a more automated fashion so that there\\'s a higher likelihood of there being some highly relevant examples for any runtime input.Single-turn v.s. multi-turn examplesAnother dimension to think about when generating examples is what the example is actually showing.The simplest types of examples just have a user input and an expected model output. These are single-turn examples.One more complex type if example is where the example is an entire conversation, usually in which a model initially responds incorrectly and a user then tells the model how to correct its answer.\\nThis is called a multi-turn example. Multi-turn examples can be useful for more nuanced tasks where its useful to show common errors and spell out exactly why they\\'re wrong and what should be done instead.2. Number of examples‚ÄãOnce we have a dataset of examples, we need to think about how many examples should be in each prompt.\\nThe key tradeoff is that more examples generally improve performance, but larger prompts increase costs and latency.\\nAnd beyond some threshold having too many examples can start to confuse the model.\\nFinding the right number of examples is highly dependent on the model, the task, the quality of the examples, and your cost and latency constraints.\\nAnecdotally, the better the model is the fewer examples it needs to perform well and the more quickly you hit steeply diminishing returns on adding more examples.\\nBut, the best/only way to reliably answer this question is to run some experiments with different numbers of examples.3. Selecting examples‚ÄãAssuming we are not adding our entire example dataset into each prompt, we need to have a way of selecting examples from our dataset based on a given input. We can do this:RandomlyBy (semantic or keyword-based) similarity of the inputsBased on some other constraints, like token sizeLangChain has a number of ExampleSelectors which make it easy to use any of these techniques.Generally, selecting by semantic similarity leads to the best model performance. But how important this is is again model and task specific, and is something worth experimenting with.4. Formatting examples‚ÄãMost state-of-the-art models these days are chat models, so we\\'ll focus on formatting examples for those. Our basic options are to insert the examples:In the system prompt as a stringAs their own messagesIf we insert our examples into the system prompt as a string, we\\'ll need to make sure it\\'s clear to the model where each example begins and which parts are the input versus output. Different models respond better to different syntaxes, like ChatML, XML, TypeScript, etc.If we insert our examples as messages, where each example is represented as a sequence of Human, AI messages, we might want to also assign names to our messages like \"example_user\" and \"example_assistant\" to make it clear that these messages correspond to different actors than the latest input message.Formatting tool call examplesOne area where formatting examples as messages can be tricky is when our example outputs have tool calls. This is because different models have different constraints on what types of message sequences are allowed when any tool calls are generated.Some models require that any AIMessage with tool calls be immediately followed by ToolMessages for every tool call,Some models additionally require that any ToolMessages be immediately followed by an AIMessage before the next HumanMessage,Some models require that tools are passed in to the model if there are any tool calls / ToolMessages in the chat history.These requirements are model-specific and should be checked for the model you are using. If your model requires ToolMessages after tool calls and/or AIMessages after ToolMessages and your examples only include expected tool calls and not the actual tool outputs, you can try adding dummy ToolMessages / AIMessages to the end of each example with generic contents to satisfy the API constraints.\\nIn these cases it\\'s especially worth experimenting with inserting your examples as strings versus messages, as having dummy messages can adversely affect certain models.You can see a case study of how Anthropic and OpenAI respond to different few-shot prompting techniques on two different tool calling benchmarks here.Retrieval‚ÄãLLMs are trained on a large but fixed dataset, limiting their ability to reason over private or recent information. Fine-tuning an LLM with specific facts is one way to mitigate this, but is often poorly suited for factual recall and can be costly.\\nRetrieval is the process of providing relevant information to an LLM to improve its response for a given input. Retrieval augmented generation (RAG) is the process of grounding the LLM generation (output) using the retrieved information.tipSee our RAG from Scratch code and video series.For a high-level guide on retrieval, see this tutorial on RAG.RAG is only as good as the retrieved documents‚Äô relevance and quality. Fortunately, an emerging set of techniques can be employed to design and improve RAG systems. We\\'ve focused on taxonomizing and summarizing many of these techniques (see below figure) and will share some high-level strategic guidance in the following sections.\\nYou can and should experiment with using different pieces together. You might also find this LangSmith guide useful for showing how to evaluate different iterations of your app.Query Translation‚ÄãFirst, consider the user input(s) to your RAG system. Ideally, a RAG system can handle a wide range of inputs, from poorly worded questions to complex multi-part queries.\\nUsing an LLM to review and optionally modify the input is the central idea behind query translation. This serves as a general buffer, optimizing raw user inputs for your retrieval system.\\nFor example, this can be as simple as extracting keywords or as complex as generating multiple sub-questions for a complex query.NameWhen to useDescriptionMulti-queryWhen you need to cover multiple perspectives of a question.Rewrite the user question from multiple perspectives, retrieve documents for each rewritten question, return the unique documents for all queries.DecompositionWhen a question can be broken down into smaller subproblems.Decompose a question into a set of subproblems / questions, which can either be solved sequentially (use the answer from first + retrieval to answer the second) or in parallel (consolidate each answer into final answer).Step-backWhen a higher-level conceptual understanding is required.First prompt the LLM to ask a generic step-back question about higher-level concepts or principles, and retrieve relevant facts about them. Use this grounding to help answer the user question.HyDEIf you have challenges retrieving relevant documents using the raw user inputs.Use an LLM to convert questions into hypothetical documents that answer the question. Use the embedded hypothetical documents to retrieve real documents with the premise that doc-doc similarity search can produce more relevant matches.tipSee our RAG from Scratch videos for a few different specific approaches:Multi-queryDecompositionStep-backHyDERouting‚ÄãSecond, consider the data sources available to your RAG system. You want to query across more than one database or across structured and unstructured data sources. Using an LLM to review the input and route it to the appropriate data source is a simple and effective approach for querying across sources.NameWhen to useDescriptionLogical routingWhen you can prompt an LLM with rules to decide where to route the input.Logical routing can use an LLM to reason about the query and choose which datastore is most appropriate.Semantic routingWhen semantic similarity is an effective way to determine where to route the input.Semantic routing embeds both query and, typically a set of prompts. It then chooses the appropriate prompt based upon similarity.tipSee our RAG from Scratch video on routing.  Query Construction‚ÄãThird, consider whether any of your data sources require specific query formats. Many structured databases use SQL. Vector stores often have specific syntax for applying keyword filters to document metadata. Using an LLM to convert a natural language query into a query syntax is a popular and powerful approach.\\nIn particular, text-to-SQL, text-to-Cypher, and query analysis for metadata filters are useful ways to interact with structured, graph, and vector databases respectively. NameWhen to UseDescriptionText to SQLIf users are asking questions that require information housed in a relational database, accessible via SQL.This uses an LLM to transform user input into a SQL query.Text-to-CypherIf users are asking questions that require information housed in a graph database, accessible via Cypher.This uses an LLM to transform user input into a Cypher query.Self QueryIf users are asking questions that are better answered by fetching documents based on metadata rather than similarity with the text.This uses an LLM to transform user input into two things: (1) a string to look up semantically, (2) a metadata filter to go along with it. This is useful because oftentimes questions are about the METADATA of documents (not the content itself).tipSee our blog post overview and RAG from Scratch video on query construction, the process of text-to-DSL where DSL is a domain specific language required to interact with a given database. This converts user questions into structured queries. Indexing‚ÄãFourth, consider the design of your document index. A simple and powerful idea is to decouple the documents that you index for retrieval from the documents that you pass to the LLM for generation. Indexing frequently uses embedding models with vector stores, which compress the semantic information in documents to fixed-size vectors.Many RAG approaches focus on splitting documents into chunks and retrieving some number based on similarity to an input question for the LLM. But chunk size and chunk number can be difficult to set and affect results if they do not provide full context for the LLM to answer a question. Furthermore, LLMs are increasingly capable of processing millions of tokens. Two approaches can address this tension: (1) Multi Vector retriever using an LLM to translate documents into any form (e.g., often into a summary) that is well-suited for indexing, but returns full documents to the LLM for generation. (2) ParentDocument retriever embeds document chunks, but also returns full documents. The idea is to get the best of both worlds: use concise representations (summaries or chunks) for retrieval, but use the full documents for answer generation.NameIndex TypeUses an LLMWhen to UseDescriptionVector storeVector storeNoIf you are just getting started and looking for something quick and easy.This is the simplest method and the one that is easiest to get started with. It involves creating embeddings for each piece of text.ParentDocumentVector store + Document StoreNoIf your pages have lots of smaller pieces of distinct information that are best indexed by themselves, but best retrieved all together.This involves indexing multiple chunks for each document. Then you find the chunks that are most similar in embedding space, but you retrieve the whole parent document and return that (rather than individual chunks).Multi VectorVector store + Document StoreSometimes during indexingIf you are able to extract information from documents that you think is more relevant to index than the text itself.This involves creating multiple vectors for each document. Each vector could be created in a myriad of ways - examples include summaries of the text and hypothetical questions.Time-Weighted Vector storeVector storeNoIf you have timestamps associated with your documents, and you want to retrieve the most recent onesThis fetches documents based on a combination of semantic similarity (as in normal vector retrieval) and recency (looking at timestamps of indexed documents)tipSee our RAG from Scratch video on indexing fundamentalsSee our RAG from Scratch video on multi vector retrieverFifth, consider ways to improve the quality of your similarity search itself. Embedding models compress text into fixed-length (vector) representations that capture the semantic content of the document. This compression is useful for search / retrieval, but puts a heavy burden on that single vector representation to capture the semantic nuance / detail of the document. In some cases, irrelevant or redundant content can dilute the semantic usefulness of the embedding.ColBERT is an interesting approach to address this with a higher granularity embeddings: (1) produce a contextually influenced embedding for each token in the document and query, (2) score similarity between each query token and all document tokens, (3) take the max, (4) do this for all query tokens, and (5) take the sum of the max scores (in step 3) for all query tokens to get a query-document similarity score; this token-wise scoring can yield strong results. There are some additional tricks to improve the quality of your retrieval. Embeddings excel at capturing semantic information, but may struggle with keyword-based queries. Many vector stores offer built-in hybrid-search to combine keyword and semantic similarity, which marries the benefits of both approaches. Furthermore, many vector stores have maximal marginal relevance, which attempts to diversify the results of a search to avoid returning similar and redundant documents. NameWhen to useDescriptionColBERTWhen higher granularity embeddings are needed.ColBERT uses contextually influenced embeddings for each token in the document and query to get a granular query-document similarity score.Hybrid searchWhen combining keyword-based and semantic similarity.Hybrid search combines keyword and semantic similarity, marrying the benefits of both approaches.Maximal Marginal Relevance (MMR)When needing to diversify search results.MMR attempts to diversify the results of a search to avoid returning similar and redundant documents.tipSee our RAG from Scratch video on ColBERT.Post-processing‚ÄãSixth, consider ways to filter or rank retrieved documents. This is very useful if you are combining documents returned from multiple sources, since it can can down-rank less relevant documents and / or compress similar documents. NameIndex TypeUses an LLMWhen to UseDescriptionContextual CompressionAnySometimesIf you are finding that your retrieved documents contain too much irrelevant information and are distracting the LLM.This puts a post-processing step on top of another retriever and extracts only the most relevant information from retrieved documents. This can be done with embeddings or an LLM.EnsembleAnyNoIf you have multiple retrieval methods and want to try combining them.This fetches documents from multiple retrievers and then combines them.Re-rankingAnyYesIf you want to rank retrieved documents based upon relevance, especially if you want to combine results from multiple retrieval methods .Given a query and a list of documents, Rerank indexes the documents from most to least semantically relevant to the query.tipSee our RAG from Scratch video on RAG-Fusion, on approach for post-processing across multiple queries:  Rewrite the user question from multiple perspectives, retrieve documents for each rewritten question, and combine the ranks of multiple search result lists to produce a single, unified ranking with Reciprocal Rank Fusion (RRF).Generation‚ÄãFinally, consider ways to build self-correction into your RAG system. RAG systems can suffer from low quality retrieval (e.g., if a user question is out of the domain for the index) and / or hallucinations in generation. A naive retrieve-generate pipeline has no ability to detect or self-correct from these kinds of errors. The concept of \"flow engineering\" has been introduced in the context of code generation: iteratively build an answer to a code question with unit tests to check and self-correct errors. Several works have applied this RAG, such as Self-RAG and Corrective-RAG. In both cases, checks for document relevance, hallucinations, and / or answer quality are performed in the RAG answer generation flow.We\\'ve found that graphs are a great way to reliably express logical flows and have implemented ideas from several of these papers using LangGraph, as shown in the figure below (red - routing, blue - fallback, green - self-correction):Routing: Adaptive RAG (paper). Route questions to different retrieval approaches, as discussed above Fallback: Corrective RAG (paper). Fallback to web search if docs are not relevant to querySelf-correction: Self-RAG (paper). Fix answers w/ hallucinations or don‚Äôt address questionNameWhen to useDescriptionSelf-RAGWhen needing to fix answers with hallucinations or irrelevant content.Self-RAG performs checks for document relevance, hallucinations, and answer quality during the RAG answer generation flow, iteratively building an answer and self-correcting errors.Corrective-RAGWhen needing a fallback mechanism for low relevance docs.Corrective-RAG includes a fallback (e.g., to web search) if the retrieved documents are not relevant to the query, ensuring higher quality and more relevant retrieval.tipSee several videos and cookbooks showcasing RAG with LangGraph: LangGraph Corrective RAGLangGraph combining Adaptive, Self-RAG, and Corrective RAG Cookbooks for RAG using LangGraphSee our LangGraph RAG recipes with partners:MetaMistralText splitting‚ÄãLangChain offers many different types of text splitters.\\nThese all live in the langchain-text-splitters package.Table columns:Name: Name of the text splitterClasses: Classes that implement this text splitterSplits On: How this text splitter splits textAdds Metadata: Whether or not this text splitter adds metadata about where each chunk came from.Description: Description of the splitter, including recommendation on when to use it.NameClassesSplits OnAdds MetadataDescriptionRecursiveRecursiveCharacterTextSplitter, RecursiveJsonSplitterA list of user defined charactersRecursively splits text. This splitting is trying to keep related pieces of text next to each other. This is the recommended way to start splitting text.HTMLHTMLHeaderTextSplitter, HTMLSectionSplitterHTML specific characters‚úÖSplits text based on HTML-specific characters. Notably, this adds in relevant information about where that chunk came from (based on the HTML)MarkdownMarkdownHeaderTextSplitter,Markdown specific characters‚úÖSplits text based on Markdown-specific characters. Notably, this adds in relevant information about where that chunk came from (based on the Markdown)Codemany languagesCode (Python, JS) specific charactersSplits text based on characters specific to coding languages. 15 different languages are available to choose from.Tokenmany classesTokensSplits text on tokens. There exist a few different ways to measure tokens.CharacterCharacterTextSplitterA user defined characterSplits text based on a user defined character. One of the simpler methods.Semantic Chunker (Experimental)SemanticChunkerSentencesFirst splits on sentences. Then combines ones next to each other if they are semantically similar enough. Taken from Greg KamradtIntegration: AI21 SemanticAI21SemanticTextSplitter‚úÖIdentifies distinct topics that form coherent pieces of text and splits along those.Evaluation‚ÄãEvaluation is the process of assessing the performance and effectiveness of your LLM-powered applications.\\nIt involves testing the model\\'s responses against a set of predefined criteria or benchmarks to ensure it meets the desired quality standards and fulfills the intended purpose.\\nThis process is vital for building reliable applications.LangSmith helps with this process in a few ways:It makes it easier to create and curate datasets via its tracing and annotation featuresIt provides an evaluation framework that helps you define metrics and run your app against your datasetIt allows you to track results over time and automatically run your evaluators on a schedule or as part of CI/CodeTo learn more, check out this LangSmith guide.Tracing‚ÄãA trace is essentially a series of steps that your application takes to go from input to output.\\nTraces contain individual steps called runs. These can be individual calls from a model, retriever,\\ntool, or sub-chains.\\nTracing gives you observability inside your chains and agents, and is vital in diagnosing issues.For a deeper dive, check out this LangSmith conceptual guide.Edit this pageWas this page helpful?You can also leave detailed feedback on GitHub.PreviousHow to create and query vector storesNextOverview of v0.2Architecturelangchain-corePartner packageslangchainlangchain-communitylanggraphlangserveLangSmithLangChain Expression Language (LCEL)Runnable interfaceComponentsChat modelsLLMsMessagesPrompt templatesExample selectorsOutput parsersChat historyDocumentsDocument loadersText splittersEmbedding modelsVector storesRetrieversKey-value storesToolsToolkitsAgentsCallbacksTechniquesStreamingFunction/tool callingStructured outputFew-shot promptingRetrievalText splittingEvaluationTracingCommunityTwitterGitHubOrganizationPythonJS/TSMoreHomepageBlogYouTubeCopyright ¬© 2024 LangChain, Inc.\\n\\n\\n\\n'"],"application/vnd.google.colaboratory.intrinsic+json":{"type":"string"}},"metadata":{},"execution_count":4}]},{"cell_type":"code","source":["from langchain_core.prompts import ChatPromptTemplate\n","from langchain_core.pydantic_v1 import BaseModel, Field\n","from langchain_openai import ChatOpenAI\n","\n","### OpenAI\n","\n","# Grader prompt\n","code_gen_prompt = ChatPromptTemplate.from_messages(\n","    [\n","        (\n","            \"system\",\n","            \"\"\"You are a coding assistant with expertise in LCEL, LangChain expression language. \\n\n","    Here is a full set of LCEL documentation:  \\n ------- \\n  {context} \\n ------- \\n Answer the user\n","    question based on the above provided documentation. Ensure any code you provide can be executed \\n\n","    with all required imports and variables defined. Structure your answer with a description of the code solution. \\n\n","    Then list the imports. And finally list the functioning code block. Please ensure to use the latest OpenAI model, gpt-4o-mini.\n","  Here is the user question:\"\"\",\n","        ),\n","        (\"placeholder\", \"{messages}\"),\n","    ]\n",")\n","\n","# Data model\n","class code(BaseModel):\n","    \"\"\"Code output\"\"\"\n","    prefix: str = Field(description=\"Description of the problem and approach\")\n","    imports: str = Field(description=\"Code block import statements\")\n","    code: str = Field(description=\"Code block not including import statements\")\n","    description = \"Schema for code solutions to questions about LCEL.\"\n","\n","expt_llm = \"gpt-4o-mini\"\n","llm = ChatOpenAI(temperature=0, model=expt_llm)\n","code_gen_chain = code_gen_prompt | llm.with_structured_output(code)\n","# question = \"How do I build a RAG chain in LCEL?\"\n","# solution = code_gen_chain.invoke({\"context\":concatenated_content,\"messages\":[(\"user\",question)]})"],"metadata":{"id":"UZsCIzm38NYb"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Test\n","question = \"How do I build a RAG chain in LCEL?\"\n","solution = code_gen_chain.invoke(\n","    {\"context\": concatenated_content, \"messages\": [(\"user\", question)]}\n",")\n","solution"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"6edb-_mU_wc_","executionInfo":{"status":"ok","timestamp":1724805244745,"user_tz":-480,"elapsed":7360,"user":{"displayName":"李辉","userId":"12972001611808140221"}},"outputId":"98117941-31f5-4bc0-b3c9-25b451048d9f"},"execution_count":null,"outputs":[{"output_type":"execute_result","data":{"text/plain":["code(prefix='This code demonstrates how to build a Retrieval-Augmented Generation (RAG) chain using LangChain Expression Language (LCEL). The RAG chain combines a retriever to fetch relevant documents and a language model to generate responses based on those documents.', imports='from langchain_core.prompts import ChatPromptTemplate\\nfrom langchain_openai import ChatOpenAI\\nfrom langchain_core.output_parsers import StrOutputParser\\nfrom langchain_core.retrievers import VectorStoreRetriever\\nfrom langchain_core.vectorstores import FAISS\\nfrom langchain_core.embeddings import OpenAIEmbeddings\\n\\n# Initialize the OpenAI model\\nmodel = ChatOpenAI(model=\"gpt-4o-mini\")\\n\\n# Create a vector store and retriever\\nembeddings = OpenAIEmbeddings()\\nvector_store = FAISS(embedding_function=embeddings)\\nretriever = VectorStoreRetriever(vector_store=vector_store)', code='# Define the prompt template for the RAG chain\\nprompt = ChatPromptTemplate.from_template(\"Given the following documents, answer the question: {question}\")\\n\\n# Create the RAG chain by combining the prompt, retriever, and model\\nrag_chain = prompt | retriever | model | StrOutputParser()', description=\"This code sets up a RAG chain that retrieves relevant documents based on a user's question and then generates a response using the OpenAI model.\")"]},"metadata":{},"execution_count":6}]},{"cell_type":"code","source":["from typing import List, TypedDict\n","\n","\n","class GraphState(TypedDict):\n","    \"\"\"\n","    Represents the state of our graph.\n","\n","    Attributes:\n","        error : Binary flag for control flow to indicate whether test error was tripped\n","        messages : With user question, error messages, reasoning\n","        generation : Code solution\n","        iterations : Number of tries\n","    \"\"\"\n","\n","    error: str\n","    messages: List\n","    generation: str\n","    iterations: int"],"metadata":{"id":"B7OHpSOH_75C"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["from langchain_core.pydantic_v1 import BaseModel, Field\n","\n","### Parameter\n","\n","# Max tries\n","max_iterations = 3\n","# Reflect\n","# flag = 'reflect'\n","flag = \"do not reflect\"\n","\n","### Nodes\n","def generate(state: GraphState):\n","    \"\"\"\n","    Generate a code solution\n","    Args:\n","        state (dict): The current graph state\n","    Returns:\n","        state (dict): New key added to state, generation\n","    \"\"\"\n","\n","    print(\"---GENERATING CODE SOLUTION---\")\n","\n","    # State\n","    messages = state[\"messages\"]\n","    iterations = state[\"iterations\"]\n","    error = state[\"error\"]\n","\n","    # We have been routed back to generation with an error\n","    if error == \"yes\":\n","        messages += [\n","            (\n","                \"user\",\n","                \"Now, try again. Invoke the code tool to structure the output with a prefix, imports, and code block:\",\n","            )\n","        ]\n","\n","    # Solution\n","    code_solution = code_gen_chain.invoke(\n","        {\"context\": concatenated_content, \"messages\": messages}\n","    )\n","    messages += [\n","        (\n","            \"assistant\",\n","            f\"{code_solution.prefix} \\n Imports: {code_solution.imports} \\n Code: {code_solution.code}\",\n","        )\n","    ]\n","\n","    # Increment\n","    iterations = iterations + 1\n","    return {\"generation\": code_solution, \"messages\": messages, \"iterations\": iterations}\n","\n","\n","def code_check(state: GraphState):\n","    \"\"\"\n","    Check code\n","\n","    Args:\n","        state (dict): The current graph state\n","\n","    Returns:\n","        state (dict): New key added to state, error\n","    \"\"\"\n","\n","    print(\"---CHECKING CODE---\")\n","\n","    # State\n","    messages = state[\"messages\"]\n","    code_solution = state[\"generation\"]\n","    iterations = state[\"iterations\"]\n","\n","    # Get solution components\n","    imports = code_solution.imports\n","    code = code_solution.code\n","\n","    # Check imports\n","    try:\n","        exec(imports)\n","    except Exception as e:\n","        print(\"---CODE IMPORT CHECK: FAILED---\")\n","        error_message = [(\"user\", f\"Your solution failed the import test: {e}\")]\n","        messages += error_message\n","        return {\n","            \"generation\": code_solution,\n","            \"messages\": messages,\n","            \"iterations\": iterations,\n","            \"error\": \"yes\",\n","        }\n","\n","    # Check execution\n","    try:\n","        exec(imports + \"\\n\" + code)\n","    except Exception as e:\n","        print(\"---CODE BLOCK CHECK: FAILED---\")\n","        error_message = [(\"user\", f\"Your solution failed the code execution test: {e}\")]\n","        messages += error_message\n","        return {\n","            \"generation\": code_solution,\n","            \"messages\": messages,\n","            \"iterations\": iterations,\n","            \"error\": \"yes\",\n","        }\n","\n","    # No errors\n","    print(\"---NO CODE TEST FAILURES---\")\n","    return {\n","        \"generation\": code_solution,\n","        \"messages\": messages,\n","        \"iterations\": iterations,\n","        \"error\": \"no\",\n","    }\n","\n","\n","def reflect(state: GraphState):\n","    \"\"\"\n","    Reflect on errors\n","\n","    Args:\n","        state (dict): The current graph state\n","\n","    Returns:\n","        state (dict): New key added to state, generation\n","    \"\"\"\n","\n","    print(\"---GENERATING CODE SOLUTION---\")\n","\n","    # State\n","    messages = state[\"messages\"]\n","    iterations = state[\"iterations\"]\n","    code_solution = state[\"generation\"]\n","\n","    # Prompt reflection\n","\n","    # Add reflection\n","    reflections = code_gen_chain.invoke(\n","        {\"context\": concatenated_content, \"messages\": messages}\n","    )\n","    messages += [(\"assistant\", f\"Here are reflections on the error: {reflections}\")]\n","    return {\"generation\": code_solution, \"messages\": messages, \"iterations\": iterations}\n","\n","\n","### Edges\n","def decide_to_finish(state: GraphState):\n","    \"\"\"\n","    Determines whether to finish.\n","\n","    Args:\n","        state (dict): The current graph state\n","\n","    Returns:\n","        str: Next node to call\n","    \"\"\"\n","    error = state[\"error\"]\n","    iterations = state[\"iterations\"]\n","\n","    if error == \"no\" or iterations == max_iterations:\n","        print(\"---DECISION: FINISH---\")\n","        return \"end\"\n","    else:\n","        print(\"---DECISION: RE-TRY SOLUTION---\")\n","        if flag == \"reflect\":\n","            return \"reflect\"\n","        else:\n","            return \"generate\""],"metadata":{"id":"sE12pjRJAAL3"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["from langgraph.graph import END, StateGraph, START\n","\n","workflow = StateGraph(GraphState)\n","\n","# Define the nodes\n","workflow.add_node(\"generate\", generate)  # generation solution\n","workflow.add_node(\"check_code\", code_check)  # check code\n","workflow.add_node(\"reflect\", reflect)  # reflect\n","\n","# Build graph\n","workflow.add_edge(START, \"generate\")\n","workflow.add_edge(\"generate\", \"check_code\")\n","workflow.add_conditional_edges(\n","    \"check_code\",\n","    decide_to_finish,\n","    {\n","        \"end\": END,\n","        \"reflect\": \"reflect\",\n","        \"generate\": \"generate\",\n","    },\n",")\n","workflow.add_edge(\"reflect\", \"generate\")\n","app = workflow.compile()"],"metadata":{"id":"lS0v97AcARz5"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["question = \"How can I directly pass a string to a runnable and use it to construct the input needed for my prompt?\"\n","app.invoke({\"messages\": [(\"user\", question)], \"iterations\": 0})"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"3Q3Fe-MdMeBh","executionInfo":{"status":"ok","timestamp":1724808055971,"user_tz":-480,"elapsed":8467,"user":{"displayName":"李辉","userId":"12972001611808140221"}},"outputId":"939ae5ee-7d47-4139-926f-0a767f30193c"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["---GENERATING CODE SOLUTION---\n","---CHECKING CODE---\n","content='Why was the cat sitting on the computer?\\n\\nBecause it wanted to keep an eye on the mouse!' additional_kwargs={'refusal': None} response_metadata={'token_usage': {'completion_tokens': 20, 'prompt_tokens': 14, 'total_tokens': 34}, 'model_name': 'gpt-4o-mini-2024-07-18', 'system_fingerprint': 'fp_f33667828e', 'finish_reason': 'stop', 'logprobs': None} id='run-89120007-af3c-481b-8835-23324858ce9d-0' usage_metadata={'input_tokens': 14, 'output_tokens': 20, 'total_tokens': 34}\n","---NO CODE TEST FAILURES---\n","---DECISION: FINISH---\n"]},{"output_type":"execute_result","data":{"text/plain":["{'error': 'no',\n"," 'messages': [('user',\n","   'How can I directly pass a string to a runnable and use it to construct the input needed for my prompt?'),\n","  ('assistant',\n","   'To directly pass a string to a runnable and use it to construct the input for a prompt in LangChain, you can utilize the `Runnable` interface. This allows you to create a prompt template that takes a string input and formats it accordingly. Below is an example demonstrating how to achieve this using the latest OpenAI model, `gpt-4o-mini`. \\n Imports: from langchain_core.prompts import PromptTemplate\\nfrom langchain_openai import ChatOpenAI\\n\\n# Define the prompt template\\nprompt_template = PromptTemplate.from_template(\"\"\"Tell me a joke about {topic}.\"\"\")\\n\\n# Initialize the chat model\\nmodel = ChatOpenAI(model=\"gpt-4o-mini\")\\n\\n# Create a runnable chain that combines the prompt and model\\nchain = prompt_template | model \\n Code: # Define the topic string\\ninput_string = \"cats\"\\n\\n# Invoke the chain with the input string\\nresponse = chain.invoke({\\'topic\\': input_string})\\n\\n# Print the response\\nprint(response)')],\n"," 'generation': code(prefix='To directly pass a string to a runnable and use it to construct the input for a prompt in LangChain, you can utilize the `Runnable` interface. This allows you to create a prompt template that takes a string input and formats it accordingly. Below is an example demonstrating how to achieve this using the latest OpenAI model, `gpt-4o-mini`.', imports='from langchain_core.prompts import PromptTemplate\\nfrom langchain_openai import ChatOpenAI\\n\\n# Define the prompt template\\nprompt_template = PromptTemplate.from_template(\"\"\"Tell me a joke about {topic}.\"\"\")\\n\\n# Initialize the chat model\\nmodel = ChatOpenAI(model=\"gpt-4o-mini\")\\n\\n# Create a runnable chain that combines the prompt and model\\nchain = prompt_template | model', code='# Define the topic string\\ninput_string = \"cats\"\\n\\n# Invoke the chain with the input string\\nresponse = chain.invoke({\\'topic\\': input_string})\\n\\n# Print the response\\nprint(response)', description=\"This code defines a prompt template that takes a topic as input, initializes a chat model, and constructs a runnable chain. It then invokes the chain with a specific topic string, in this case, 'cats', and prints the response.\"),\n"," 'iterations': 1}"]},"metadata":{},"execution_count":18}]},{"cell_type":"code","source":["from langchain_core.prompts import PromptTemplate\n","from langchain_openai import ChatOpenAI\n","\n","# Define the prompt template\n","prompt_template = PromptTemplate.from_template(\"\"\"Tell me a joke about {topic}.\"\"\")\n","# Initialize the chat model\n","model = ChatOpenAI(model=\"gpt-4o-mini\")\n","# Create a runnable chain that combines the prompt and model\n","chain = prompt_template | model\n","# Define the topic string\n","input_string = \"cats\"\n","# Invoke the chain with the input string\n","response = chain.invoke({'topic': input_string})\n","# Print the response\n","print(response)"],"metadata":{"colab":{"base_uri":"https://localhost:8080/"},"id":"sI_9-tlwknp8","executionInfo":{"status":"ok","timestamp":1724808237614,"user_tz":-480,"elapsed":1649,"user":{"displayName":"李辉","userId":"12972001611808140221"}},"outputId":"63acd5c7-4ade-4957-a7b3-50840a4c56b6"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["content='Why was the cat sitting on the computer?\\n\\nBecause it wanted to keep an eye on the mouse!' additional_kwargs={'refusal': None} response_metadata={'token_usage': {'completion_tokens': 20, 'prompt_tokens': 14, 'total_tokens': 34}, 'model_name': 'gpt-4o-mini-2024-07-18', 'system_fingerprint': 'fp_f33667828e', 'finish_reason': 'stop', 'logprobs': None} id='run-140c9d74-f321-4f19-adf3-1833d89a7520-0' usage_metadata={'input_tokens': 14, 'output_tokens': 20, 'total_tokens': 34}\n"]}]},{"cell_type":"markdown","source":["# 测试集"],"metadata":{"id":"H0Y4C-nOnrWJ"}},{"cell_type":"code","source":["import langsmith\n","client = langsmith.Client()"],"metadata":{"id":"gmv6mqaQAWQm"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Clone the dataset to your tenant to use it\n","public_dataset = (\n","    \"https://smith.langchain.com/public/326674a6-62bd-462d-88ae-eea49d503f9d/d\"\n",")\n","client.clone_public_dataset(public_dataset)"],"metadata":{"id":"s_zGmvqDAYtB"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["from langsmith.schemas import Example, Run\n","\n","\n","def check_import(run: Run, example: Example) -> dict:\n","    imports = run.outputs.get(\"imports\")\n","    try:\n","        exec(imports)\n","        return {\"key\": \"import_check\", \"score\": 1}\n","    except Exception:\n","        return {\"key\": \"import_check\", \"score\": 0}\n","\n","\n","def check_execution(run: Run, example: Example) -> dict:\n","    imports = run.outputs.get(\"imports\")\n","    code = run.outputs.get(\"code\")\n","    try:\n","        exec(imports + \"\\n\" + code)\n","        return {\"key\": \"code_execution_check\", \"score\": 1}\n","    except Exception:\n","        return {\"key\": \"code_execution_check\", \"score\": 0}"],"metadata":{"id":"0FnTtyENBcn_"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["def predict_base_case(example: dict):\n","    \"\"\"Context stuffing\"\"\"\n","    solution = code_gen_chain.invoke(\n","        {\"context\": concatenated_content, \"messages\": [(\"user\", example[\"question\"])]}\n","    )\n","    solution_structured = code_gen_chain.invoke([(\"code\", solution)])\n","    return {\"imports\": solution_structured.imports, \"code\": solution_structured.code}\n","\n","\n","def predict_langgraph(example: dict):\n","    \"\"\"LangGraph\"\"\"\n","    graph = app.invoke({\"messages\": [(\"user\", example[\"question\"])], \"iterations\": 0})\n","    solution = graph[\"generation\"]\n","    return {\"imports\": solution.imports, \"code\": solution.code}"],"metadata":{"id":"sGErZjXgBxAD"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["from langsmith.evaluation import evaluate\n","\n","# Evaluator\n","code_evalulator = [check_import, check_execution]\n","\n","# Dataset\n","dataset_name = \"lcel-teacher-eval\""],"metadata":{"id":"muKqqOYgBfqy"},"execution_count":null,"outputs":[]},{"cell_type":"code","source":["# Run base case\n","experiment_results_ = evaluate(\n","    predict_base_case,\n","    data=dataset_name,\n","    evaluators=code_evalulator,\n","    experiment_prefix=f\"test-without-langgraph-{expt_llm}\",\n","    max_concurrency=2,\n","    metadata={\n","        \"llm\": expt_llm,\n","    },\n",")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":295,"referenced_widgets":["ee48e9b394774ff8b244f678a3352cf4","b7002ee8d40342d19210dcc548c27aa4","856aa125e113453bbd026f20ee843908","522f69cbc1bf4e5d88b4fd97a4d219a4","888e6958ac8b4d2d858a9688e16dd830","896c46701ef942ad87d0116602744dbc","40435e30966f44efa7fea1c84b054e09","f083317683424036951bd56a6e307167","e5b809b105854b349b0252be952c4e95","4a1cfb40a74c45969ee1475e2376ee1c","aad2458fc6a24e6a8f5038647be8bde5"]},"id":"OOFxH2vEB1GT","executionInfo":{"status":"ok","timestamp":1722215703465,"user_tz":-480,"elapsed":101743,"user":{"displayName":"李辉","userId":"12972001611808140221"}},"outputId":"50734521-a2a8-4cfa-9793-fb2e9dbca199"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["View the evaluation results for experiment: 'test-without-langgraph-gpt-4o-mini-8d53e367' at:\n","https://smith.langchain.com/o/97965422-56ca-5e42-94ee-b511c4b3c617/datasets/ad69a8e2-ee48-454d-aee6-c4b4a437e1b7/compare?selectedSessions=5fe0a6ea-21cc-4434-9b27-c63d19297a21\n","\n","\n"]},{"output_type":"display_data","data":{"text/plain":["0it [00:00, ?it/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"ee48e9b394774ff8b244f678a3352cf4"}},"metadata":{}},{"output_type":"stream","name":"stderr","text":["ERROR:langsmith.evaluation._runner:Error running target function: Error code: 428 - {'error': {'message': '已触发低积分多请求限制(15RPM),请充值或者过1分钟后重试', 'type': 'openai_hk_error'}}\n","ERROR:langsmith.evaluation._runner:Error running target function: Error code: 428 - {'error': {'message': '已触发低积分多请求限制(15RPM),请充值或者过1分钟后重试', 'type': 'openai_hk_error'}}\n","ERROR:langsmith.evaluation._runner:Error running target function: Error code: 428 - {'error': {'message': '已触发低积分多请求限制(15RPM),请充值或者过1分钟后重试', 'type': 'openai_hk_error'}}\n","ERROR:langsmith.evaluation._runner:Error running target function: Error code: 428 - {'error': {'message': '已触发低积分多请求限制(15RPM),请充值或者过1分钟后重试', 'type': 'openai_hk_error'}}\n","ERROR:langsmith.evaluation._runner:Error running target function: Error code: 428 - {'error': {'message': '已触发低积分多请求限制(15RPM),请充值或者过1分钟后重试', 'type': 'openai_hk_error'}}\n","ERROR:langsmith.evaluation._runner:Error running target function: Error code: 428 - {'error': {'message': '已触发低积分多请求限制(15RPM),请充值或者过1分钟后重试', 'type': 'openai_hk_error'}}\n","ERROR:langsmith.evaluation._runner:Error running target function: Error code: 428 - {'error': {'message': '已触发低积分多请求限制(15RPM),请充值或者过1分钟后重试', 'type': 'openai_hk_error'}}\n"]},{"output_type":"stream","name":"stdout","text":["<class 'pydantic.v1.main.PromptInput'>\n","Please answer the question: 'where did harrison work' in italian.\n"]}]},{"cell_type":"code","source":["# Run with langgraph\n","experiment_results = evaluate(\n","    predict_langgraph,\n","    data=dataset_name,\n","    evaluators=code_evalulator,\n","    experiment_prefix=f\"test-with-langgraph-{expt_llm}-{flag}\",\n","    max_concurrency=2,\n","    metadata={\n","        \"llm\": expt_llm,\n","        \"feedback\": flag,\n","    },\n",")"],"metadata":{"colab":{"base_uri":"https://localhost:8080/","height":1000,"referenced_widgets":["546b22a963f54805960c31a10c1ad369","3581bd030e4349fd8eb740ba0052d353","fca139cfd6e5451ea214624fe965b7ce","0ce92fcac1dc4b7fac64c21d28fcaced","f88b7dc9c1ad49f2aaf19fb9920260cc","19809f7164fd4dedaef209d293c8f6a8","1b5a77df387c47869c71fb283b9502bc","42229b1d4a904fa895873eeb09b118bd","2586da556b7045f0880eb73b8302fe6d","8948252684574d2596efeb4d519ec253","5ec9591d10eb446b8ed2fe87b5220b63"]},"id":"rpkBYJ_qCAsT","executionInfo":{"status":"ok","timestamp":1722216246528,"user_tz":-480,"elapsed":74084,"user":{"displayName":"李辉","userId":"12972001611808140221"}},"outputId":"24456b17-a03a-42ba-b3c6-e101ef2d723a"},"execution_count":null,"outputs":[{"output_type":"stream","name":"stdout","text":["View the evaluation results for experiment: 'test-with-langgraph-gpt-4o-mini-do not reflect-ea442c35' at:\n","https://smith.langchain.com/o/97965422-56ca-5e42-94ee-b511c4b3c617/datasets/ad69a8e2-ee48-454d-aee6-c4b4a437e1b7/compare?selectedSessions=b708b5f0-6fd5-4a3b-9746-9aac9f943257\n","\n","\n"]},{"output_type":"display_data","data":{"text/plain":["0it [00:00, ?it/s]"],"application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"546b22a963f54805960c31a10c1ad369"}},"metadata":{}},{"output_type":"stream","name":"stdout","text":["---GENERATING CODE SOLUTION---\n","---GENERATING CODE SOLUTION---\n","---CHECKING CODE---\n","---CHECKING CODE---\n","---CODE IMPORT CHECK: FAILED---\n","---DECISION: RE-TRY SOLUTION---\n","---GENERATING CODE SOLUTION---\n","content='Why was the cat sitting on the computer?\\n\\nBecause it wanted to keep an eye on the mouse!' response_metadata={'token_usage': {'completion_tokens': 20, 'prompt_tokens': 13, 'total_tokens': 33}, 'model_name': 'gpt-4o-mini-2024-07-18', 'system_fingerprint': 'fp_ba606877f9', 'finish_reason': 'stop', 'logprobs': None} id='run-56aabff9-c8c4-4b0b-8867-76b3012bc08f-0' usage_metadata={'input_tokens': 13, 'output_tokens': 20, 'total_tokens': 33}\n","---NO CODE TEST FAILURES---\n","---DECISION: FINISH---\n","---GENERATING CODE SOLUTION---\n","---CHECKING CODE---\n","---CODE IMPORT CHECK: FAILED---\n","---DECISION: RE-TRY SOLUTION---\n","---GENERATING CODE SOLUTION---\n","---CHECKING CODE---\n","---CODE IMPORT CHECK: FAILED---\n","---DECISION: RE-TRY SOLUTION---\n","---GENERATING CODE SOLUTION---\n"]},{"output_type":"stream","name":"stderr","text":["ERROR:langsmith.evaluation._runner:Error running target function: 1 validation error for code\n","prefix\n","  field required (type=value_error.missing)\n"]},{"output_type":"stream","name":"stdout","text":["---GENERATING CODE SOLUTION---\n","---CHECKING CODE---\n","---CODE IMPORT CHECK: FAILED---\n","---DECISION: RE-TRY SOLUTION---\n","---GENERATING CODE SOLUTION---\n","---CHECKING CODE---\n","---CODE IMPORT CHECK: FAILED---\n","---DECISION: RE-TRY SOLUTION---\n","---GENERATING CODE SOLUTION---\n","---CHECKING CODE---\n","---CODE BLOCK CHECK: FAILED---\n","---DECISION: FINISH---\n","---GENERATING CODE SOLUTION---\n","---CHECKING CODE---\n","---CODE IMPORT CHECK: FAILED---\n","---DECISION: RE-TRY SOLUTION---\n","---GENERATING CODE SOLUTION---\n","---CHECKING CODE---\n","---CODE IMPORT CHECK: FAILED---\n","---DECISION: RE-TRY SOLUTION---\n","---GENERATING CODE SOLUTION---\n","---CHECKING CODE---\n","---CODE IMPORT CHECK: FAILED---\n","---DECISION: FINISH---\n","---GENERATING CODE SOLUTION---\n","---CHECKING CODE---\n","---CODE IMPORT CHECK: FAILED---\n","---DECISION: RE-TRY SOLUTION---\n","---GENERATING CODE SOLUTION---\n","---CHECKING CODE---\n","---CODE IMPORT CHECK: FAILED---\n","---DECISION: RE-TRY SOLUTION---\n","---GENERATING CODE SOLUTION---\n","---CHECKING CODE---\n","---CODE IMPORT CHECK: FAILED---\n","---DECISION: FINISH---\n","---GENERATING CODE SOLUTION---\n","---CHECKING CODE---\n"]},{"output_type":"stream","name":"stderr","text":["/usr/local/lib/python3.10/dist-packages/langchain_core/_api/deprecation.py:139: LangChainDeprecationWarning: The class `OpenAI` was deprecated in LangChain 0.0.10 and will be removed in 0.3.0. An updated version of the class exists in the langchain-openai package and should be used instead. To use it run `pip install -U langchain-openai` and import as `from langchain_openai import OpenAI`.\n","  warn_deprecated(\n","/usr/local/lib/python3.10/dist-packages/langchain_core/_api/deprecation.py:139: LangChainDeprecationWarning: The class `LLMChain` was deprecated in LangChain 0.1.17 and will be removed in 1.0. Use RunnableSequence, e.g., `prompt | llm` instead.\n","  warn_deprecated(\n"]},{"output_type":"stream","name":"stdout","text":["---CODE BLOCK CHECK: FAILED---\n","---DECISION: RE-TRY SOLUTION---\n","---GENERATING CODE SOLUTION---\n"]},{"output_type":"stream","name":"stderr","text":["ERROR:langsmith.evaluation._runner:Error running target function: Error code: 428 - {'error': {'message': '已触发低积分多请求限制(15RPM),请充值或者过1分钟后重试', 'type': 'openai_hk_error'}}\n"]},{"output_type":"stream","name":"stdout","text":["---GENERATING CODE SOLUTION---\n"]},{"output_type":"stream","name":"stderr","text":["ERROR:langsmith.evaluation._runner:Error running target function: Error code: 428 - {'error': {'message': '已触发低积分多请求限制(15RPM),请充值或者过1分钟后重试', 'type': 'openai_hk_error'}}\n","ERROR:langsmith.evaluation._runner:Error running target function: Error code: 428 - {'error': {'message': '已触发低积分多请求限制(15RPM),请充值或者过1分钟后重试', 'type': 'openai_hk_error'}}\n"]},{"output_type":"stream","name":"stdout","text":["---GENERATING CODE SOLUTION---\n","---GENERATING CODE SOLUTION---\n"]},{"output_type":"stream","name":"stderr","text":["ERROR:langsmith.evaluation._runner:Error running target function: Error code: 428 - {'error': {'message': '已触发低积分多请求限制(15RPM),请充值或者过1分钟后重试', 'type': 'openai_hk_error'}}\n","ERROR:langsmith.evaluation._runner:Error running target function: Error code: 428 - {'error': {'message': '已触发低积分多请求限制(15RPM),请充值或者过1分钟后重试', 'type': 'openai_hk_error'}}\n"]},{"output_type":"stream","name":"stdout","text":["---GENERATING CODE SOLUTION---\n","---GENERATING CODE SOLUTION---\n"]},{"output_type":"stream","name":"stderr","text":["ERROR:langsmith.evaluation._runner:Error running target function: Error code: 428 - {'error': {'message': '已触发低积分多请求限制(15RPM),请充值或者过1分钟后重试', 'type': 'openai_hk_error'}}\n","ERROR:langsmith.evaluation._runner:Error running target function: Error code: 428 - {'error': {'message': '已触发低积分多请求限制(15RPM),请充值或者过1分钟后重试', 'type': 'openai_hk_error'}}\n"]},{"output_type":"stream","name":"stdout","text":["---GENERATING CODE SOLUTION---\n","---GENERATING CODE SOLUTION---\n"]},{"output_type":"stream","name":"stderr","text":["ERROR:langsmith.evaluation._runner:Error running target function: Error code: 428 - {'error': {'message': '已触发低积分多请求限制(15RPM),请充值或者过1分钟后重试', 'type': 'openai_hk_error'}}\n","ERROR:langsmith.evaluation._runner:Error running target function: Error code: 428 - {'error': {'message': '已触发低积分多请求限制(15RPM),请充值或者过1分钟后重试', 'type': 'openai_hk_error'}}\n"]},{"output_type":"stream","name":"stdout","text":["---GENERATING CODE SOLUTION---\n","---GENERATING CODE SOLUTION---\n"]},{"output_type":"stream","name":"stderr","text":["ERROR:langsmith.evaluation._runner:Error running target function: Error code: 428 - {'error': {'message': '已触发低积分多请求限制(15RPM),请充值或者过1分钟后重试', 'type': 'openai_hk_error'}}\n","ERROR:langsmith.evaluation._runner:Error running target function: Error code: 428 - {'error': {'message': '已触发低积分多请求限制(15RPM),请充值或者过1分钟后重试', 'type': 'openai_hk_error'}}\n"]},{"output_type":"stream","name":"stdout","text":["---GENERATING CODE SOLUTION---\n","---GENERATING CODE SOLUTION---\n"]},{"output_type":"stream","name":"stderr","text":["ERROR:langsmith.evaluation._runner:Error running target function: Error code: 428 - {'error': {'message': '已触发低积分多请求限制(15RPM),请充值或者过1分钟后重试', 'type': 'openai_hk_error'}}\n","ERROR:langsmith.evaluation._runner:Error running target function: Error code: 428 - {'error': {'message': '已触发低积分多请求限制(15RPM),请充值或者过1分钟后重试', 'type': 'openai_hk_error'}}\n"]},{"output_type":"stream","name":"stdout","text":["---GENERATING CODE SOLUTION---\n","---GENERATING CODE SOLUTION---\n"]},{"output_type":"stream","name":"stderr","text":["ERROR:langsmith.evaluation._runner:Error running target function: Error code: 428 - {'error': {'message': '已触发低积分多请求限制(15RPM),请充值或者过1分钟后重试', 'type': 'openai_hk_error'}}\n"]},{"output_type":"stream","name":"stdout","text":["---CHECKING CODE---\n","---CODE BLOCK CHECK: FAILED---\n","---DECISION: RE-TRY SOLUTION---\n","---GENERATING CODE SOLUTION---\n"]},{"output_type":"stream","name":"stderr","text":["ERROR:langsmith.evaluation._runner:Error running target function: Error code: 428 - {'error': {'message': '已触发低积分多请求限制(15RPM),请充值或者过1分钟后重试', 'type': 'openai_hk_error'}}\n"]}]}]}